git.oblomov.eu Git - linux-2.6/blob - drivers/cpufreq/cpufreq.c

   1 /*
   2  *  linux/drivers/cpufreq/cpufreq.c
   3  *
   4  *  Copyright (C) 2001 Russell King
   5  *            (C) 2002 - 2003 Dominik Brodowski <linux@brodo.de>
   6  *
   7  *  Oct 2005 - Ashok Raj <ashok.raj@intel.com>
   8  *      Added handling for CPU hotplug
   9  *  Feb 2006 - Jacob Shin <jacob.shin@amd.com>
  10  *      Fix handling for CPU hotplug -- affected CPUs
  11  *
  12  * This program is free software; you can redistribute it and/or modify
  13  * it under the terms of the GNU General Public License version 2 as
  14  * published by the Free Software Foundation.
  15  *
  16  */
  17
  18 #include <linux/kernel.h>
  19 #include <linux/module.h>
  20 #include <linux/init.h>
  21 #include <linux/notifier.h>
  22 #include <linux/cpufreq.h>
  23 #include <linux/delay.h>
  24 #include <linux/interrupt.h>
  25 #include <linux/spinlock.h>
  26 #include <linux/device.h>
  27 #include <linux/slab.h>
  28 #include <linux/cpu.h>
  29 #include <linux/completion.h>
  30 #include <linux/mutex.h>
  31
  32 #define dprintk(msg...) cpufreq_debug_printk(CPUFREQ_DEBUG_CORE, \
  33                                                 "cpufreq-core", msg)
  34
  35 /**
  36  * The "cpufreq driver" - the arch- or hardware-dependent low
  37  * level driver of CPUFreq support, and its spinlock. This lock
  38  * also protects the cpufreq_cpu_data array.
  39  */
  40 static struct cpufreq_driver *cpufreq_driver;
  41 static DEFINE_PER_CPU(struct cpufreq_policy *, cpufreq_cpu_data);
  42 #ifdef CONFIG_HOTPLUG_CPU
  43 /* This one keeps track of the previously set governor of a removed CPU */
  44 static DEFINE_PER_CPU(struct cpufreq_governor *, cpufreq_cpu_governor);
  45 #endif
  46 static DEFINE_SPINLOCK(cpufreq_driver_lock);
  47
  48 /*
  49  * cpu_policy_rwsem is a per CPU reader-writer semaphore designed to cure
  50  * all cpufreq/hotplug/workqueue/etc related lock issues.
  51  *
  52  * The rules for this semaphore:
  53  * - Any routine that wants to read from the policy structure will
  54  *   do a down_read on this semaphore.
  55  * - Any routine that will write to the policy structure and/or may take away
  56  *   the policy altogether (eg. CPU hotplug), will hold this lock in write
  57  *   mode before doing so.
  58  *
  59  * Additional rules:
  60  * - All holders of the lock should check to make sure that the CPU they
  61  *   are concerned with are online after they get the lock.
  62  * - Governor routines that can be called in cpufreq hotplug path should not
  63  *   take this sem as top level hotplug notifier handler takes this.
  64  */
  65 static DEFINE_PER_CPU(int, policy_cpu);
  66 static DEFINE_PER_CPU(struct rw_semaphore, cpu_policy_rwsem);
  67
  68 #define lock_policy_rwsem(mode, cpu)                                    \
  69 int lock_policy_rwsem_##mode                                            \
  70 (int cpu)                                                               \
  71 {                                                                       \
  72         int policy_cpu = per_cpu(policy_cpu, cpu);                      \
  73         BUG_ON(policy_cpu == -1);                                       \
  74         down_##mode(&per_cpu(cpu_policy_rwsem, policy_cpu));            \
  75         if (unlikely(!cpu_online(cpu))) {                               \
  76                 up_##mode(&per_cpu(cpu_policy_rwsem, policy_cpu));      \
  77                 return -1;                                              \
  78         }                                                               \
  79                                                                         \
  80         return 0;                                                       \
  81 }
  82
  83 lock_policy_rwsem(read, cpu);
  84 EXPORT_SYMBOL_GPL(lock_policy_rwsem_read);
  85
  86 lock_policy_rwsem(write, cpu);
  87 EXPORT_SYMBOL_GPL(lock_policy_rwsem_write);
  88
  89 void unlock_policy_rwsem_read(int cpu)
  90 {
  91         int policy_cpu = per_cpu(policy_cpu, cpu);
  92         BUG_ON(policy_cpu == -1);
  93         up_read(&per_cpu(cpu_policy_rwsem, policy_cpu));
  94 }
  95 EXPORT_SYMBOL_GPL(unlock_policy_rwsem_read);
  96
  97 void unlock_policy_rwsem_write(int cpu)
  98 {
  99         int policy_cpu = per_cpu(policy_cpu, cpu);
 100         BUG_ON(policy_cpu == -1);
 101         up_write(&per_cpu(cpu_policy_rwsem, policy_cpu));
 102 }
 103 EXPORT_SYMBOL_GPL(unlock_policy_rwsem_write);
 104
 105
 106 /* internal prototypes */
 107 static int __cpufreq_governor(struct cpufreq_policy *policy,
 108                 unsigned int event);
 109 static unsigned int __cpufreq_get(unsigned int cpu);
 110 static void handle_update(struct work_struct *work);
 111
 112 /**
 113  * Two notifier lists: the "policy" list is involved in the
 114  * validation process for a new CPU frequency policy; the
 115  * "transition" list for kernel code that needs to handle
 116  * changes to devices when the CPU clock speed changes.
 117  * The mutex locks both lists.
 118  */
 119 static BLOCKING_NOTIFIER_HEAD(cpufreq_policy_notifier_list);
 120 static struct srcu_notifier_head cpufreq_transition_notifier_list;
 121
 122 static bool init_cpufreq_transition_notifier_list_called;
 123 static int __init init_cpufreq_transition_notifier_list(void)
 124 {
 125         srcu_init_notifier_head(&cpufreq_transition_notifier_list);
 126         init_cpufreq_transition_notifier_list_called = true;
 127         return 0;
 128 }
 129 pure_initcall(init_cpufreq_transition_notifier_list);
 130
 131 static LIST_HEAD(cpufreq_governor_list);
 132 static DEFINE_MUTEX(cpufreq_governor_mutex);
 133
 134 struct cpufreq_policy *cpufreq_cpu_get(unsigned int cpu)
 135 {
 136         struct cpufreq_policy *data;
 137         unsigned long flags;
 138
 139         if (cpu >= nr_cpu_ids)
 140                 goto err_out;
 141
 142         /* get the cpufreq driver */
 143         spin_lock_irqsave(&cpufreq_driver_lock, flags);
 144
 145         if (!cpufreq_driver)
 146                 goto err_out_unlock;
 147
 148         if (!try_module_get(cpufreq_driver->owner))
 149                 goto err_out_unlock;
 150
 151
 152         /* get the CPU */
 153         data = per_cpu(cpufreq_cpu_data, cpu);
 154
 155         if (!data)
 156                 goto err_out_put_module;
 157
 158         if (!kobject_get(&data->kobj))
 159                 goto err_out_put_module;
 160
 161         spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
 162         return data;
 163
 164 err_out_put_module:
 165         module_put(cpufreq_driver->owner);
 166 err_out_unlock:
 167         spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
 168 err_out:
 169         return NULL;
 170 }
 171 EXPORT_SYMBOL_GPL(cpufreq_cpu_get);
 172
 173
 174 void cpufreq_cpu_put(struct cpufreq_policy *data)
 175 {
 176         kobject_put(&data->kobj);
 177         module_put(cpufreq_driver->owner);
 178 }
 179 EXPORT_SYMBOL_GPL(cpufreq_cpu_put);
 180
 181
 182 /*********************************************************************
 183  *                     UNIFIED DEBUG HELPERS                         *
 184  *********************************************************************/
 185 #ifdef CONFIG_CPU_FREQ_DEBUG
 186
 187 /* what part(s) of the CPUfreq subsystem are debugged? */
 188 static unsigned int debug;
 189
 190 /* is the debug output ratelimit'ed using printk_ratelimit? User can
 191  * set or modify this value.
 192  */
 193 static unsigned int debug_ratelimit = 1;
 194
 195 /* is the printk_ratelimit'ing enabled? It's enabled after a successful
 196  * loading of a cpufreq driver, temporarily disabled when a new policy
 197  * is set, and disabled upon cpufreq driver removal
 198  */
 199 static unsigned int disable_ratelimit = 1;
 200 static DEFINE_SPINLOCK(disable_ratelimit_lock);
 201
 202 static void cpufreq_debug_enable_ratelimit(void)
 203 {
 204         unsigned long flags;
 205
 206         spin_lock_irqsave(&disable_ratelimit_lock, flags);
 207         if (disable_ratelimit)
 208                 disable_ratelimit--;
 209         spin_unlock_irqrestore(&disable_ratelimit_lock, flags);
 210 }
 211
 212 static void cpufreq_debug_disable_ratelimit(void)
 213 {
 214         unsigned long flags;
 215
 216         spin_lock_irqsave(&disable_ratelimit_lock, flags);
 217         disable_ratelimit++;
 218         spin_unlock_irqrestore(&disable_ratelimit_lock, flags);
 219 }
 220
 221 void cpufreq_debug_printk(unsigned int type, const char *prefix,
 222                         const char *fmt, ...)
 223 {
 224         char s[256];
 225         va_list args;
 226         unsigned int len;
 227         unsigned long flags;
 228
 229         WARN_ON(!prefix);
 230         if (type & debug) {
 231                 spin_lock_irqsave(&disable_ratelimit_lock, flags);
 232                 if (!disable_ratelimit && debug_ratelimit
 233                                         && !printk_ratelimit()) {
 234                         spin_unlock_irqrestore(&disable_ratelimit_lock, flags);
 235                         return;
 236                 }
 237                 spin_unlock_irqrestore(&disable_ratelimit_lock, flags);
 238
 239                 len = snprintf(s, 256, KERN_DEBUG "%s: ", prefix);
 240
 241                 va_start(args, fmt);
 242                 len += vsnprintf(&s[len], (256 - len), fmt, args);
 243                 va_end(args);
 244
 245                 printk(s);
 246
 247                 WARN_ON(len < 5);
 248         }
 249 }
 250 EXPORT_SYMBOL(cpufreq_debug_printk);
 251
 252
 253 module_param(debug, uint, 0644);
 254 MODULE_PARM_DESC(debug, "CPUfreq debugging: add 1 to debug core,"
 255                         " 2 to debug drivers, and 4 to debug governors.");
 256
 257 module_param(debug_ratelimit, uint, 0644);
 258 MODULE_PARM_DESC(debug_ratelimit, "CPUfreq debugging:"
 259                                         " set to 0 to disable ratelimiting.");
 260
 261 #else /* !CONFIG_CPU_FREQ_DEBUG */
 262
 263 static inline void cpufreq_debug_enable_ratelimit(void) { return; }
 264 static inline void cpufreq_debug_disable_ratelimit(void) { return; }
 265
 266 #endif /* CONFIG_CPU_FREQ_DEBUG */
 267
 268
 269 /*********************************************************************
 270  *            EXTERNALLY AFFECTING FREQUENCY CHANGES                 *
 271  *********************************************************************/
 272
 273 /**
 274  * adjust_jiffies - adjust the system "loops_per_jiffy"
 275  *
 276  * This function alters the system "loops_per_jiffy" for the clock
 277  * speed change. Note that loops_per_jiffy cannot be updated on SMP
 278  * systems as each CPU might be scaled differently. So, use the arch
 279  * per-CPU loops_per_jiffy value wherever possible.
 280  */
 281 #ifndef CONFIG_SMP
 282 static unsigned long l_p_j_ref;
 283 static unsigned int  l_p_j_ref_freq;
 284
 285 static void adjust_jiffies(unsigned long val, struct cpufreq_freqs *ci)
 286 {
 287         if (ci->flags & CPUFREQ_CONST_LOOPS)
 288                 return;
 289
 290         if (!l_p_j_ref_freq) {
 291                 l_p_j_ref = loops_per_jiffy;
 292                 l_p_j_ref_freq = ci->old;
 293                 dprintk("saving %lu as reference value for loops_per_jiffy; "
 294                         "freq is %u kHz\n", l_p_j_ref, l_p_j_ref_freq);
 295         }
 296         if ((val == CPUFREQ_PRECHANGE  && ci->old < ci->new) ||
 297             (val == CPUFREQ_POSTCHANGE && ci->old > ci->new) ||
 298             (val == CPUFREQ_RESUMECHANGE || val == CPUFREQ_SUSPENDCHANGE)) {
 299                 loops_per_jiffy = cpufreq_scale(l_p_j_ref, l_p_j_ref_freq,
 300                                                                 ci->new);
 301                 dprintk("scaling loops_per_jiffy to %lu "
 302                         "for frequency %u kHz\n", loops_per_jiffy, ci->new);
 303         }
 304 }
 305 #else
 306 static inline void adjust_jiffies(unsigned long val, struct cpufreq_freqs *ci)
 307 {
 308         return;
 309 }
 310 #endif
 311
 312
 313 /**
 314  * cpufreq_notify_transition - call notifier chain and adjust_jiffies
 315  * on frequency transition.
 316  *
 317  * This function calls the transition notifiers and the "adjust_jiffies"
 318  * function. It is called twice on all CPU frequency changes that have
 319  * external effects.
 320  */
 321 void cpufreq_notify_transition(struct cpufreq_freqs *freqs, unsigned int state)
 322 {
 323         struct cpufreq_policy *policy;
 324
 325         BUG_ON(irqs_disabled());
 326
 327         freqs->flags = cpufreq_driver->flags;
 328         dprintk("notification %u of frequency transition to %u kHz\n",
 329                 state, freqs->new);
 330
 331         policy = per_cpu(cpufreq_cpu_data, freqs->cpu);
 332         switch (state) {
 333
 334         case CPUFREQ_PRECHANGE:
 335                 /* detect if the driver reported a value as "old frequency"
 336                  * which is not equal to what the cpufreq core thinks is
 337                  * "old frequency".
 338                  */
 339                 if (!(cpufreq_driver->flags & CPUFREQ_CONST_LOOPS)) {
 340                         if ((policy) && (policy->cpu == freqs->cpu) &&
 341                             (policy->cur) && (policy->cur != freqs->old)) {
 342                                 dprintk("Warning: CPU frequency is"
 343                                         " %u, cpufreq assumed %u kHz.\n",
 344                                         freqs->old, policy->cur);
 345                                 freqs->old = policy->cur;
 346                         }
 347                 }
 348                 srcu_notifier_call_chain(&cpufreq_transition_notifier_list,
 349                                 CPUFREQ_PRECHANGE, freqs);
 350                 adjust_jiffies(CPUFREQ_PRECHANGE, freqs);
 351                 break;
 352
 353         case CPUFREQ_POSTCHANGE:
 354                 adjust_jiffies(CPUFREQ_POSTCHANGE, freqs);
 355                 srcu_notifier_call_chain(&cpufreq_transition_notifier_list,
 356                                 CPUFREQ_POSTCHANGE, freqs);
 357                 if (likely(policy) && likely(policy->cpu == freqs->cpu))
 358                         policy->cur = freqs->new;
 359                 break;
 360         }
 361 }
 362 EXPORT_SYMBOL_GPL(cpufreq_notify_transition);
 363
 364
 365
 366 /*********************************************************************
 367  *                          SYSFS INTERFACE                          *
 368  *********************************************************************/
 369
 370 static struct cpufreq_governor *__find_governor(const char *str_governor)
 371 {
 372         struct cpufreq_governor *t;
 373
 374         list_for_each_entry(t, &cpufreq_governor_list, governor_list)
 375                 if (!strnicmp(str_governor, t->name, CPUFREQ_NAME_LEN))
 376                         return t;
 377
 378         return NULL;
 379 }
 380
 381 /**
 382  * cpufreq_parse_governor - parse a governor string
 383  */
 384 static int cpufreq_parse_governor(char *str_governor, unsigned int *policy,
 385                                 struct cpufreq_governor **governor)
 386 {
 387         int err = -EINVAL;
 388
 389         if (!cpufreq_driver)
 390                 goto out;
 391
 392         if (cpufreq_driver->setpolicy) {
 393                 if (!strnicmp(str_governor, "performance", CPUFREQ_NAME_LEN)) {
 394                         *policy = CPUFREQ_POLICY_PERFORMANCE;
 395                         err = 0;
 396                 } else if (!strnicmp(str_governor, "powersave",
 397                                                 CPUFREQ_NAME_LEN)) {
 398                         *policy = CPUFREQ_POLICY_POWERSAVE;
 399                         err = 0;
 400                 }
 401         } else if (cpufreq_driver->target) {
 402                 struct cpufreq_governor *t;
 403
 404                 mutex_lock(&cpufreq_governor_mutex);
 405
 406                 t = __find_governor(str_governor);
 407
 408                 if (t == NULL) {
 409                         char *name = kasprintf(GFP_KERNEL, "cpufreq_%s",
 410                                                                 str_governor);
 411
 412                         if (name) {
 413                                 int ret;
 414
 415                                 mutex_unlock(&cpufreq_governor_mutex);
 416                                 ret = request_module("%s", name);
 417                                 mutex_lock(&cpufreq_governor_mutex);
 418
 419                                 if (ret == 0)
 420                                         t = __find_governor(str_governor);
 421                         }
 422
 423                         kfree(name);
 424                 }
 425
 426                 if (t != NULL) {
 427                         *governor = t;
 428                         err = 0;
 429                 }
 430
 431                 mutex_unlock(&cpufreq_governor_mutex);
 432         }
 433 out:
 434         return err;
 435 }
 436
 437
 438 /**
 439  * cpufreq_per_cpu_attr_read() / show_##file_name() -
 440  * print out cpufreq information
 441  *
 442  * Write out information from cpufreq_driver->policy[cpu]; object must be
 443  * "unsigned int".
 444  */
 445
 446 #define show_one(file_name, object)                     \
 447 static ssize_t show_##file_name                         \
 448 (struct cpufreq_policy *policy, char *buf)              \
 449 {                                                       \
 450         return sprintf(buf, "%u\n", policy->object);    \
 451 }
 452
 453 show_one(cpuinfo_min_freq, cpuinfo.min_freq);
 454 show_one(cpuinfo_max_freq, cpuinfo.max_freq);
 455 show_one(cpuinfo_transition_latency, cpuinfo.transition_latency);
 456 show_one(scaling_min_freq, min);
 457 show_one(scaling_max_freq, max);
 458 show_one(scaling_cur_freq, cur);
 459
 460 static int __cpufreq_set_policy(struct cpufreq_policy *data,
 461                                 struct cpufreq_policy *policy);
 462
 463 /**
 464  * cpufreq_per_cpu_attr_write() / store_##file_name() - sysfs write access
 465  */
 466 #define store_one(file_name, object)                    \
 467 static ssize_t store_##file_name                                        \
 468 (struct cpufreq_policy *policy, const char *buf, size_t count)          \
 469 {                                                                       \
 470         unsigned int ret = -EINVAL;                                     \
 471         struct cpufreq_policy new_policy;                               \
 472                                                                         \
 473         ret = cpufreq_get_policy(&new_policy, policy->cpu);             \
 474         if (ret)                                                        \
 475                 return -EINVAL;                                         \
 476                                                                         \
 477         ret = sscanf(buf, "%u", &new_policy.object);                    \
 478         if (ret != 1)                                                   \
 479                 return -EINVAL;                                         \
 480                                                                         \
 481         ret = __cpufreq_set_policy(policy, &new_policy);                \
 482         policy->user_policy.object = policy->object;                    \
 483                                                                         \
 484         return ret ? ret : count;                                       \
 485 }
 486
 487 store_one(scaling_min_freq, min);
 488 store_one(scaling_max_freq, max);
 489
 490 /**
 491  * show_cpuinfo_cur_freq - current CPU frequency as detected by hardware
 492  */
 493 static ssize_t show_cpuinfo_cur_freq(struct cpufreq_policy *policy,
 494                                         char *buf)
 495 {
 496         unsigned int cur_freq = __cpufreq_get(policy->cpu);
 497         if (!cur_freq)
 498                 return sprintf(buf, "<unknown>");
 499         return sprintf(buf, "%u\n", cur_freq);
 500 }
 501
 502
 503 /**
 504  * show_scaling_governor - show the current policy for the specified CPU
 505  */
 506 static ssize_t show_scaling_governor(struct cpufreq_policy *policy, char *buf)
 507 {
 508         if (policy->policy == CPUFREQ_POLICY_POWERSAVE)
 509                 return sprintf(buf, "powersave\n");
 510         else if (policy->policy == CPUFREQ_POLICY_PERFORMANCE)
 511                 return sprintf(buf, "performance\n");
 512         else if (policy->governor)
 513                 return scnprintf(buf, CPUFREQ_NAME_LEN, "%s\n",
 514                                 policy->governor->name);
 515         return -EINVAL;
 516 }
 517
 518
 519 /**
 520  * store_scaling_governor - store policy for the specified CPU
 521  */
 522 static ssize_t store_scaling_governor(struct cpufreq_policy *policy,
 523                                         const char *buf, size_t count)
 524 {
 525         unsigned int ret = -EINVAL;
 526         char    str_governor[16];
 527         struct cpufreq_policy new_policy;
 528
 529         ret = cpufreq_get_policy(&new_policy, policy->cpu);
 530         if (ret)
 531                 return ret;
 532
 533         ret = sscanf(buf, "%15s", str_governor);
 534         if (ret != 1)
 535                 return -EINVAL;
 536
 537         if (cpufreq_parse_governor(str_governor, &new_policy.policy,
 538                                                 &new_policy.governor))
 539                 return -EINVAL;
 540
 541         /* Do not use cpufreq_set_policy here or the user_policy.max
 542            will be wrongly overridden */
 543         ret = __cpufreq_set_policy(policy, &new_policy);
 544
 545         policy->user_policy.policy = policy->policy;
 546         policy->user_policy.governor = policy->governor;
 547
 548         if (ret)
 549                 return ret;
 550         else
 551                 return count;
 552 }
 553
 554 /**
 555  * show_scaling_driver - show the cpufreq driver currently loaded
 556  */
 557 static ssize_t show_scaling_driver(struct cpufreq_policy *policy, char *buf)
 558 {
 559         return scnprintf(buf, CPUFREQ_NAME_LEN, "%s\n", cpufreq_driver->name);
 560 }
 561
 562 /**
 563  * show_scaling_available_governors - show the available CPUfreq governors
 564  */
 565 static ssize_t show_scaling_available_governors(struct cpufreq_policy *policy,
 566                                                 char *buf)
 567 {
 568         ssize_t i = 0;
 569         struct cpufreq_governor *t;
 570
 571         if (!cpufreq_driver->target) {
 572                 i += sprintf(buf, "performance powersave");
 573                 goto out;
 574         }
 575
 576         list_for_each_entry(t, &cpufreq_governor_list, governor_list) {
 577                 if (i >= (ssize_t) ((PAGE_SIZE / sizeof(char))
 578                     - (CPUFREQ_NAME_LEN + 2)))
 579                         goto out;
 580                 i += scnprintf(&buf[i], CPUFREQ_NAME_LEN, "%s ", t->name);
 581         }
 582 out:
 583         i += sprintf(&buf[i], "\n");
 584         return i;
 585 }
 586
 587 static ssize_t show_cpus(const struct cpumask *mask, char *buf)
 588 {
 589         ssize_t i = 0;
 590         unsigned int cpu;
 591
 592         for_each_cpu(cpu, mask) {
 593                 if (i)
 594                         i += scnprintf(&buf[i], (PAGE_SIZE - i - 2), " ");
 595                 i += scnprintf(&buf[i], (PAGE_SIZE - i - 2), "%u", cpu);
 596                 if (i >= (PAGE_SIZE - 5))
 597                         break;
 598         }
 599         i += sprintf(&buf[i], "\n");
 600         return i;
 601 }
 602
 603 /**
 604  * show_related_cpus - show the CPUs affected by each transition even if
 605  * hw coordination is in use
 606  */
 607 static ssize_t show_related_cpus(struct cpufreq_policy *policy, char *buf)
 608 {
 609         if (cpumask_empty(policy->related_cpus))
 610                 return show_cpus(policy->cpus, buf);
 611         return show_cpus(policy->related_cpus, buf);
 612 }
 613
 614 /**
 615  * show_affected_cpus - show the CPUs affected by each transition
 616  */
 617 static ssize_t show_affected_cpus(struct cpufreq_policy *policy, char *buf)
 618 {
 619         return show_cpus(policy->cpus, buf);
 620 }
 621
 622 static ssize_t store_scaling_setspeed(struct cpufreq_policy *policy,
 623                                         const char *buf, size_t count)
 624 {
 625         unsigned int freq = 0;
 626         unsigned int ret;
 627
 628         if (!policy->governor || !policy->governor->store_setspeed)
 629                 return -EINVAL;
 630
 631         ret = sscanf(buf, "%u", &freq);
 632         if (ret != 1)
 633                 return -EINVAL;
 634
 635         policy->governor->store_setspeed(policy, freq);
 636
 637         return count;
 638 }
 639
 640 static ssize_t show_scaling_setspeed(struct cpufreq_policy *policy, char *buf)
 641 {
 642         if (!policy->governor || !policy->governor->show_setspeed)
 643                 return sprintf(buf, "<unsupported>\n");
 644
 645         return policy->governor->show_setspeed(policy, buf);
 646 }
 647
 648 #define define_one_ro(_name) \
 649 static struct freq_attr _name = \
 650 __ATTR(_name, 0444, show_##_name, NULL)
 651
 652 #define define_one_ro0400(_name) \
 653 static struct freq_attr _name = \
 654 __ATTR(_name, 0400, show_##_name, NULL)
 655
 656 #define define_one_rw(_name) \
 657 static struct freq_attr _name = \
 658 __ATTR(_name, 0644, show_##_name, store_##_name)
 659
 660 define_one_ro0400(cpuinfo_cur_freq);
 661 define_one_ro(cpuinfo_min_freq);
 662 define_one_ro(cpuinfo_max_freq);
 663 define_one_ro(cpuinfo_transition_latency);
 664 define_one_ro(scaling_available_governors);
 665 define_one_ro(scaling_driver);
 666 define_one_ro(scaling_cur_freq);
 667 define_one_ro(related_cpus);
 668 define_one_ro(affected_cpus);
 669 define_one_rw(scaling_min_freq);
 670 define_one_rw(scaling_max_freq);
 671 define_one_rw(scaling_governor);
 672 define_one_rw(scaling_setspeed);
 673
 674 static struct attribute *default_attrs[] = {
 675         &cpuinfo_min_freq.attr,
 676         &cpuinfo_max_freq.attr,
 677         &cpuinfo_transition_latency.attr,
 678         &scaling_min_freq.attr,
 679         &scaling_max_freq.attr,
 680         &affected_cpus.attr,
 681         &related_cpus.attr,
 682         &scaling_governor.attr,
 683         &scaling_driver.attr,
 684         &scaling_available_governors.attr,
 685         &scaling_setspeed.attr,
 686         NULL
 687 };
 688
 689 #define to_policy(k) container_of(k, struct cpufreq_policy, kobj)
 690 #define to_attr(a) container_of(a, struct freq_attr, attr)
 691
 692 static ssize_t show(struct kobject *kobj, struct attribute *attr, char *buf)
 693 {
 694         struct cpufreq_policy *policy = to_policy(kobj);
 695         struct freq_attr *fattr = to_attr(attr);
 696         ssize_t ret = -EINVAL;
 697         policy = cpufreq_cpu_get(policy->cpu);
 698         if (!policy)
 699                 goto no_policy;
 700
 701         if (lock_policy_rwsem_read(policy->cpu) < 0)
 702                 goto fail;
 703
 704         if (fattr->show)
 705                 ret = fattr->show(policy, buf);
 706         else
 707                 ret = -EIO;
 708
 709         unlock_policy_rwsem_read(policy->cpu);
 710 fail:
 711         cpufreq_cpu_put(policy);
 712 no_policy:
 713         return ret;
 714 }
 715
 716 static ssize_t store(struct kobject *kobj, struct attribute *attr,
 717                      const char *buf, size_t count)
 718 {
 719         struct cpufreq_policy *policy = to_policy(kobj);
 720         struct freq_attr *fattr = to_attr(attr);
 721         ssize_t ret = -EINVAL;
 722         policy = cpufreq_cpu_get(policy->cpu);
 723         if (!policy)
 724                 goto no_policy;
 725
 726         if (lock_policy_rwsem_write(policy->cpu) < 0)
 727                 goto fail;
 728
 729         if (fattr->store)
 730                 ret = fattr->store(policy, buf, count);
 731         else
 732                 ret = -EIO;
 733
 734         unlock_policy_rwsem_write(policy->cpu);
 735 fail:
 736         cpufreq_cpu_put(policy);
 737 no_policy:
 738         return ret;
 739 }
 740
 741 static void cpufreq_sysfs_release(struct kobject *kobj)
 742 {
 743         struct cpufreq_policy *policy = to_policy(kobj);
 744         dprintk("last reference is dropped\n");
 745         complete(&policy->kobj_unregister);
 746 }
 747
 748 static struct sysfs_ops sysfs_ops = {
 749         .show   = show,
 750         .store  = store,
 751 };
 752
 753 static struct kobj_type ktype_cpufreq = {
 754         .sysfs_ops      = &sysfs_ops,
 755         .default_attrs  = default_attrs,
 756         .release        = cpufreq_sysfs_release,
 757 };
 758
 759
 760 /**
 761  * cpufreq_add_dev - add a CPU device
 762  *
 763  * Adds the cpufreq interface for a CPU device.
 764  *
 765  * The Oracle says: try running cpufreq registration/unregistration concurrently
 766  * with with cpu hotplugging and all hell will break loose. Tried to clean this
 767  * mess up, but more thorough testing is needed. - Mathieu
 768  */
 769 static int cpufreq_add_dev(struct sys_device *sys_dev)
 770 {
 771         unsigned int cpu = sys_dev->id;
 772         int ret = 0;
 773         struct cpufreq_policy new_policy;
 774         struct cpufreq_policy *policy;
 775         struct freq_attr **drv_attr;
 776         struct sys_device *cpu_sys_dev;
 777         unsigned long flags;
 778         unsigned int j;
 779
 780         if (cpu_is_offline(cpu))
 781                 return 0;
 782
 783         cpufreq_debug_disable_ratelimit();
 784         dprintk("adding CPU %u\n", cpu);
 785
 786 #ifdef CONFIG_SMP
 787         /* check whether a different CPU already registered this
 788          * CPU because it is in the same boat. */
 789         policy = cpufreq_cpu_get(cpu);
 790         if (unlikely(policy)) {
 791                 cpufreq_cpu_put(policy);
 792                 cpufreq_debug_enable_ratelimit();
 793                 return 0;
 794         }
 795 #endif
 796
 797         if (!try_module_get(cpufreq_driver->owner)) {
 798                 ret = -EINVAL;
 799                 goto module_out;
 800         }
 801
 802         policy = kzalloc(sizeof(struct cpufreq_policy), GFP_KERNEL);
 803         if (!policy) {
 804                 ret = -ENOMEM;
 805                 goto nomem_out;
 806         }
 807         if (!alloc_cpumask_var(&policy->cpus, GFP_KERNEL)) {
 808                 ret = -ENOMEM;
 809                 goto err_free_policy;
 810         }
 811         if (!zalloc_cpumask_var(&policy->related_cpus, GFP_KERNEL)) {
 812                 ret = -ENOMEM;
 813                 goto err_free_cpumask;
 814         }
 815
 816         policy->cpu = cpu;
 817         cpumask_copy(policy->cpus, cpumask_of(cpu));
 818
 819         /* Initially set CPU itself as the policy_cpu */
 820         per_cpu(policy_cpu, cpu) = cpu;
 821         ret = (lock_policy_rwsem_write(cpu) < 0);
 822         WARN_ON(ret);
 823
 824         init_completion(&policy->kobj_unregister);
 825         INIT_WORK(&policy->update, handle_update);
 826
 827         /* Set governor before ->init, so that driver could check it */
 828         policy->governor = CPUFREQ_DEFAULT_GOVERNOR;
 829         /* call driver. From then on the cpufreq must be able
 830          * to accept all calls to ->verify and ->setpolicy for this CPU
 831          */
 832         ret = cpufreq_driver->init(policy);
 833         if (ret) {
 834                 dprintk("initialization failed\n");
 835                 goto err_unlock_policy;
 836         }
 837         policy->user_policy.min = policy->min;
 838         policy->user_policy.max = policy->max;
 839
 840         blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
 841                                      CPUFREQ_START, policy);
 842
 843 #ifdef CONFIG_SMP
 844
 845 #ifdef CONFIG_HOTPLUG_CPU
 846         if (per_cpu(cpufreq_cpu_governor, cpu)) {
 847                 policy->governor = per_cpu(cpufreq_cpu_governor, cpu);
 848                 dprintk("Restoring governor %s for cpu %d\n",
 849                        policy->governor->name, cpu);
 850         }
 851 #endif
 852
 853         for_each_cpu(j, policy->cpus) {
 854                 struct cpufreq_policy *managed_policy;
 855
 856                 if (cpu == j)
 857                         continue;
 858
 859                 /* Check for existing affected CPUs.
 860                  * They may not be aware of it due to CPU Hotplug.
 861                  */
 862                 managed_policy = cpufreq_cpu_get(j);
 863                 if (unlikely(managed_policy)) {
 864
 865                         /* Set proper policy_cpu */
 866                         unlock_policy_rwsem_write(cpu);
 867                         per_cpu(policy_cpu, cpu) = managed_policy->cpu;
 868
 869                         if (lock_policy_rwsem_write(cpu) < 0) {
 870                                 /* Should not go through policy unlock path */
 871                                 if (cpufreq_driver->exit)
 872                                         cpufreq_driver->exit(policy);
 873                                 ret = -EBUSY;
 874                                 cpufreq_cpu_put(managed_policy);
 875                                 goto err_free_cpumask;
 876                         }
 877
 878                         spin_lock_irqsave(&cpufreq_driver_lock, flags);
 879                         cpumask_copy(managed_policy->cpus, policy->cpus);
 880                         per_cpu(cpufreq_cpu_data, cpu) = managed_policy;
 881                         spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
 882
 883                         dprintk("CPU already managed, adding link\n");
 884                         ret = sysfs_create_link(&sys_dev->kobj,
 885                                                 &managed_policy->kobj,
 886                                                 "cpufreq");
 887                         if (!ret)
 888                                 cpufreq_cpu_put(managed_policy);
 889                         /*
 890                          * Success. We only needed to be added to the mask.
 891                          * Call driver->exit() because only the cpu parent of
 892                          * the kobj needed to call init().
 893                          */
 894                         goto out_driver_exit; /* call driver->exit() */
 895                 }
 896         }
 897 #endif
 898         memcpy(&new_policy, policy, sizeof(struct cpufreq_policy));
 899
 900         /* prepare interface data */
 901         ret = kobject_init_and_add(&policy->kobj, &ktype_cpufreq, &sys_dev->kobj,
 902                                    "cpufreq");
 903         if (ret)
 904                 goto out_driver_exit;
 905
 906         /* set up files for this cpu device */
 907         drv_attr = cpufreq_driver->attr;
 908         while ((drv_attr) && (*drv_attr)) {
 909                 ret = sysfs_create_file(&policy->kobj, &((*drv_attr)->attr));
 910                 if (ret)
 911                         goto err_out_kobj_put;
 912                 drv_attr++;
 913         }
 914         if (cpufreq_driver->get) {
 915                 ret = sysfs_create_file(&policy->kobj, &cpuinfo_cur_freq.attr);
 916                 if (ret)
 917                         goto err_out_kobj_put;
 918         }
 919         if (cpufreq_driver->target) {
 920                 ret = sysfs_create_file(&policy->kobj, &scaling_cur_freq.attr);
 921                 if (ret)
 922                         goto err_out_kobj_put;
 923         }
 924
 925         spin_lock_irqsave(&cpufreq_driver_lock, flags);
 926         for_each_cpu(j, policy->cpus) {
 927                 per_cpu(cpufreq_cpu_data, j) = policy;
 928                 per_cpu(policy_cpu, j) = policy->cpu;
 929         }
 930         spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
 931
 932         /* symlink affected CPUs */
 933         for_each_cpu(j, policy->cpus) {
 934                 struct cpufreq_policy *managed_policy;
 935
 936                 if (j == cpu)
 937                         continue;
 938                 if (!cpu_online(j))
 939                         continue;
 940
 941                 dprintk("CPU %u already managed, adding link\n", j);
 942                 managed_policy = cpufreq_cpu_get(cpu);
 943                 cpu_sys_dev = get_cpu_sysdev(j);
 944                 ret = sysfs_create_link(&cpu_sys_dev->kobj, &policy->kobj,
 945                                         "cpufreq");
 946                 if (ret) {
 947                         cpufreq_cpu_put(managed_policy);
 948                         goto err_out_unregister;
 949                 }
 950         }
 951
 952         policy->governor = NULL; /* to assure that the starting sequence is
 953                                   * run in cpufreq_set_policy */
 954
 955         /* set default policy */
 956         ret = __cpufreq_set_policy(policy, &new_policy);
 957         policy->user_policy.policy = policy->policy;
 958         policy->user_policy.governor = policy->governor;
 959
 960         if (ret) {
 961                 dprintk("setting policy failed\n");
 962                 goto err_out_unregister;
 963         }
 964
 965         unlock_policy_rwsem_write(cpu);
 966
 967         kobject_uevent(&policy->kobj, KOBJ_ADD);
 968         module_put(cpufreq_driver->owner);
 969         dprintk("initialization complete\n");
 970         cpufreq_debug_enable_ratelimit();
 971
 972         return 0;
 973
 974
 975 err_out_unregister:
 976         spin_lock_irqsave(&cpufreq_driver_lock, flags);
 977         for_each_cpu(j, policy->cpus)
 978                 per_cpu(cpufreq_cpu_data, j) = NULL;
 979         spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
 980
 981 err_out_kobj_put:
 982         kobject_put(&policy->kobj);
 983         wait_for_completion(&policy->kobj_unregister);
 984
 985 out_driver_exit:
 986         if (cpufreq_driver->exit)
 987                 cpufreq_driver->exit(policy);
 988
 989 err_unlock_policy:
 990         unlock_policy_rwsem_write(cpu);
 991 err_free_cpumask:
 992         free_cpumask_var(policy->cpus);
 993 err_free_policy:
 994         kfree(policy);
 995 nomem_out:
 996         module_put(cpufreq_driver->owner);
 997 module_out:
 998         cpufreq_debug_enable_ratelimit();
 999         return ret;
1000 }
1001
1002
1003 /**
1004  * __cpufreq_remove_dev - remove a CPU device
1005  *
1006  * Removes the cpufreq interface for a CPU device.
1007  * Caller should already have policy_rwsem in write mode for this CPU.
1008  * This routine frees the rwsem before returning.
1009  */
1010 static int __cpufreq_remove_dev(struct sys_device *sys_dev)
1011 {
1012         unsigned int cpu = sys_dev->id;
1013         unsigned long flags;
1014         struct cpufreq_policy *data;
1015 #ifdef CONFIG_SMP
1016         struct sys_device *cpu_sys_dev;
1017         unsigned int j;
1018 #endif
1019
1020         cpufreq_debug_disable_ratelimit();
1021         dprintk("unregistering CPU %u\n", cpu);
1022
1023         spin_lock_irqsave(&cpufreq_driver_lock, flags);
1024         data = per_cpu(cpufreq_cpu_data, cpu);
1025
1026         if (!data) {
1027                 spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
1028                 cpufreq_debug_enable_ratelimit();
1029                 unlock_policy_rwsem_write(cpu);
1030                 return -EINVAL;
1031         }
1032         per_cpu(cpufreq_cpu_data, cpu) = NULL;
1033
1034
1035 #ifdef CONFIG_SMP
1036         /* if this isn't the CPU which is the parent of the kobj, we
1037          * only need to unlink, put and exit
1038          */
1039         if (unlikely(cpu != data->cpu)) {
1040                 dprintk("removing link\n");
1041                 cpumask_clear_cpu(cpu, data->cpus);
1042                 spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
1043                 sysfs_remove_link(&sys_dev->kobj, "cpufreq");
1044                 cpufreq_cpu_put(data);
1045                 cpufreq_debug_enable_ratelimit();
1046                 unlock_policy_rwsem_write(cpu);
1047                 return 0;
1048         }
1049 #endif
1050
1051 #ifdef CONFIG_SMP
1052
1053 #ifdef CONFIG_HOTPLUG_CPU
1054         per_cpu(cpufreq_cpu_governor, cpu) = data->governor;
1055 #endif
1056
1057         /* if we have other CPUs still registered, we need to unlink them,
1058          * or else wait_for_completion below will lock up. Clean the
1059          * per_cpu(cpufreq_cpu_data) while holding the lock, and remove
1060          * the sysfs links afterwards.
1061          */
1062         if (unlikely(cpumask_weight(data->cpus) > 1)) {
1063                 for_each_cpu(j, data->cpus) {
1064                         if (j == cpu)
1065                                 continue;
1066                         per_cpu(cpufreq_cpu_data, j) = NULL;
1067                 }
1068         }
1069
1070         spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
1071
1072         if (unlikely(cpumask_weight(data->cpus) > 1)) {
1073                 for_each_cpu(j, data->cpus) {
1074                         if (j == cpu)
1075                                 continue;
1076                         dprintk("removing link for cpu %u\n", j);
1077 #ifdef CONFIG_HOTPLUG_CPU
1078                         per_cpu(cpufreq_cpu_governor, j) = data->governor;
1079 #endif
1080                         cpu_sys_dev = get_cpu_sysdev(j);
1081                         sysfs_remove_link(&cpu_sys_dev->kobj, "cpufreq");
1082                         cpufreq_cpu_put(data);
1083                 }
1084         }
1085 #else
1086         spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
1087 #endif
1088
1089         if (cpufreq_driver->target)
1090                 __cpufreq_governor(data, CPUFREQ_GOV_STOP);
1091
1092         kobject_put(&data->kobj);
1093
1094         /* we need to make sure that the underlying kobj is actually
1095          * not referenced anymore by anybody before we proceed with
1096          * unloading.
1097          */
1098         dprintk("waiting for dropping of refcount\n");
1099         wait_for_completion(&data->kobj_unregister);
1100         dprintk("wait complete\n");
1101
1102         if (cpufreq_driver->exit)
1103                 cpufreq_driver->exit(data);
1104
1105         unlock_policy_rwsem_write(cpu);
1106
1107         free_cpumask_var(data->related_cpus);
1108         free_cpumask_var(data->cpus);
1109         kfree(data);
1110         per_cpu(cpufreq_cpu_data, cpu) = NULL;
1111
1112         cpufreq_debug_enable_ratelimit();
1113         return 0;
1114 }
1115
1116
1117 static int cpufreq_remove_dev(struct sys_device *sys_dev)
1118 {
1119         unsigned int cpu = sys_dev->id;
1120         int retval;
1121
1122         if (cpu_is_offline(cpu))
1123                 return 0;
1124
1125         if (unlikely(lock_policy_rwsem_write(cpu)))
1126                 BUG();
1127
1128         retval = __cpufreq_remove_dev(sys_dev);
1129         return retval;
1130 }
1131
1132
1133 static void handle_update(struct work_struct *work)
1134 {
1135         struct cpufreq_policy *policy =
1136                 container_of(work, struct cpufreq_policy, update);
1137         unsigned int cpu = policy->cpu;
1138         dprintk("handle_update for cpu %u called\n", cpu);
1139         cpufreq_update_policy(cpu);
1140 }
1141
1142 /**
1143  *      cpufreq_out_of_sync - If actual and saved CPU frequency differs, we're in deep trouble.
1144  *      @cpu: cpu number
1145  *      @old_freq: CPU frequency the kernel thinks the CPU runs at
1146  *      @new_freq: CPU frequency the CPU actually runs at
1147  *
1148  *      We adjust to current frequency first, and need to clean up later.
1149  *      So either call to cpufreq_update_policy() or schedule handle_update()).
1150  */
1151 static void cpufreq_out_of_sync(unsigned int cpu, unsigned int old_freq,
1152                                 unsigned int new_freq)
1153 {
1154         struct cpufreq_freqs freqs;
1155
1156         dprintk("Warning: CPU frequency out of sync: cpufreq and timing "
1157                "core thinks of %u, is %u kHz.\n", old_freq, new_freq);
1158
1159         freqs.cpu = cpu;
1160         freqs.old = old_freq;
1161         freqs.new = new_freq;
1162         cpufreq_notify_transition(&freqs, CPUFREQ_PRECHANGE);
1163         cpufreq_notify_transition(&freqs, CPUFREQ_POSTCHANGE);
1164 }
1165
1166
1167 /**
1168  * cpufreq_quick_get - get the CPU frequency (in kHz) from policy->cur
1169  * @cpu: CPU number
1170  *
1171  * This is the last known freq, without actually getting it from the driver.
1172  * Return value will be same as what is shown in scaling_cur_freq in sysfs.
1173  */
1174 unsigned int cpufreq_quick_get(unsigned int cpu)
1175 {
1176         struct cpufreq_policy *policy = cpufreq_cpu_get(cpu);
1177         unsigned int ret_freq = 0;
1178
1179         if (policy) {
1180                 ret_freq = policy->cur;
1181                 cpufreq_cpu_put(policy);
1182         }
1183
1184         return ret_freq;
1185 }
1186 EXPORT_SYMBOL(cpufreq_quick_get);
1187
1188
1189 static unsigned int __cpufreq_get(unsigned int cpu)
1190 {
1191         struct cpufreq_policy *policy = per_cpu(cpufreq_cpu_data, cpu);
1192         unsigned int ret_freq = 0;
1193
1194         if (!cpufreq_driver->get)
1195                 return ret_freq;
1196
1197         ret_freq = cpufreq_driver->get(cpu);
1198
1199         if (ret_freq && policy->cur &&
1200                 !(cpufreq_driver->flags & CPUFREQ_CONST_LOOPS)) {
1201                 /* verify no discrepancy between actual and
1202                                         saved value exists */
1203                 if (unlikely(ret_freq != policy->cur)) {
1204                         cpufreq_out_of_sync(cpu, policy->cur, ret_freq);
1205                         schedule_work(&policy->update);
1206                 }
1207         }
1208
1209         return ret_freq;
1210 }
1211
1212 /**
1213  * cpufreq_get - get the current CPU frequency (in kHz)
1214  * @cpu: CPU number
1215  *
1216  * Get the CPU current (static) CPU frequency
1217  */
1218 unsigned int cpufreq_get(unsigned int cpu)
1219 {
1220         unsigned int ret_freq = 0;
1221         struct cpufreq_policy *policy = cpufreq_cpu_get(cpu);
1222
1223         if (!policy)
1224                 goto out;
1225
1226         if (unlikely(lock_policy_rwsem_read(cpu)))
1227                 goto out_policy;
1228
1229         ret_freq = __cpufreq_get(cpu);
1230
1231         unlock_policy_rwsem_read(cpu);
1232
1233 out_policy:
1234         cpufreq_cpu_put(policy);
1235 out:
1236         return ret_freq;
1237 }
1238 EXPORT_SYMBOL(cpufreq_get);
1239
1240
1241 /**
1242  *      cpufreq_suspend - let the low level driver prepare for suspend
1243  */
1244
1245 static int cpufreq_suspend(struct sys_device *sysdev, pm_message_t pmsg)
1246 {
1247         int cpu = sysdev->id;
1248         int ret = 0;
1249         unsigned int cur_freq = 0;
1250         struct cpufreq_policy *cpu_policy;
1251
1252         dprintk("suspending cpu %u\n", cpu);
1253
1254         if (!cpu_online(cpu))
1255                 return 0;
1256
1257         /* we may be lax here as interrupts are off. Nonetheless
1258          * we need to grab the correct cpu policy, as to check
1259          * whether we really run on this CPU.
1260          */
1261
1262         cpu_policy = cpufreq_cpu_get(cpu);
1263         if (!cpu_policy)
1264                 return -EINVAL;
1265
1266         /* only handle each CPU group once */
1267         if (unlikely(cpu_policy->cpu != cpu))
1268                 goto out;
1269
1270         if (cpufreq_driver->suspend) {
1271                 ret = cpufreq_driver->suspend(cpu_policy, pmsg);
1272                 if (ret) {
1273                         printk(KERN_ERR "cpufreq: suspend failed in ->suspend "
1274                                         "step on CPU %u\n", cpu_policy->cpu);
1275                         goto out;
1276                 }
1277         }
1278
1279         if (cpufreq_driver->flags & CPUFREQ_CONST_LOOPS)
1280                 goto out;
1281
1282         if (cpufreq_driver->get)
1283                 cur_freq = cpufreq_driver->get(cpu_policy->cpu);
1284
1285         if (!cur_freq || !cpu_policy->cur) {
1286                 printk(KERN_ERR "cpufreq: suspend failed to assert current "
1287                        "frequency is what timing core thinks it is.\n");
1288                 goto out;
1289         }
1290
1291         if (unlikely(cur_freq != cpu_policy->cur)) {
1292                 struct cpufreq_freqs freqs;
1293
1294                 if (!(cpufreq_driver->flags & CPUFREQ_PM_NO_WARN))
1295                         dprintk("Warning: CPU frequency is %u, "
1296                                "cpufreq assumed %u kHz.\n",
1297                                cur_freq, cpu_policy->cur);
1298
1299                 freqs.cpu = cpu;
1300                 freqs.old = cpu_policy->cur;
1301                 freqs.new = cur_freq;
1302
1303                 srcu_notifier_call_chain(&cpufreq_transition_notifier_list,
1304                                     CPUFREQ_SUSPENDCHANGE, &freqs);
1305                 adjust_jiffies(CPUFREQ_SUSPENDCHANGE, &freqs);
1306
1307                 cpu_policy->cur = cur_freq;
1308         }
1309
1310 out:
1311         cpufreq_cpu_put(cpu_policy);
1312         return ret;
1313 }
1314
1315 /**
1316  *      cpufreq_resume -  restore proper CPU frequency handling after resume
1317  *
1318  *      1.) resume CPUfreq hardware support (cpufreq_driver->resume())
1319  *      2.) if ->target and !CPUFREQ_CONST_LOOPS: verify we're in sync
1320  *      3.) schedule call cpufreq_update_policy() ASAP as interrupts are
1321  *          restored.
1322  */
1323 static int cpufreq_resume(struct sys_device *sysdev)
1324 {
1325         int cpu = sysdev->id;
1326         int ret = 0;
1327         struct cpufreq_policy *cpu_policy;
1328
1329         dprintk("resuming cpu %u\n", cpu);
1330
1331         if (!cpu_online(cpu))
1332                 return 0;
1333
1334         /* we may be lax here as interrupts are off. Nonetheless
1335          * we need to grab the correct cpu policy, as to check
1336          * whether we really run on this CPU.
1337          */
1338
1339         cpu_policy = cpufreq_cpu_get(cpu);
1340         if (!cpu_policy)
1341                 return -EINVAL;
1342
1343         /* only handle each CPU group once */
1344         if (unlikely(cpu_policy->cpu != cpu))
1345                 goto fail;
1346
1347         if (cpufreq_driver->resume) {
1348                 ret = cpufreq_driver->resume(cpu_policy);
1349                 if (ret) {
1350                         printk(KERN_ERR "cpufreq: resume failed in ->resume "
1351                                         "step on CPU %u\n", cpu_policy->cpu);
1352                         goto fail;
1353                 }
1354         }
1355
1356         if (!(cpufreq_driver->flags & CPUFREQ_CONST_LOOPS)) {
1357                 unsigned int cur_freq = 0;
1358
1359                 if (cpufreq_driver->get)
1360                         cur_freq = cpufreq_driver->get(cpu_policy->cpu);
1361
1362                 if (!cur_freq || !cpu_policy->cur) {
1363                         printk(KERN_ERR "cpufreq: resume failed to assert "
1364                                         "current frequency is what timing core "
1365                                         "thinks it is.\n");
1366                         goto out;
1367                 }
1368
1369                 if (unlikely(cur_freq != cpu_policy->cur)) {
1370                         struct cpufreq_freqs freqs;
1371
1372                         if (!(cpufreq_driver->flags & CPUFREQ_PM_NO_WARN))
1373                                 dprintk("Warning: CPU frequency "
1374                                        "is %u, cpufreq assumed %u kHz.\n",
1375                                        cur_freq, cpu_policy->cur);
1376
1377                         freqs.cpu = cpu;
1378                         freqs.old = cpu_policy->cur;
1379                         freqs.new = cur_freq;
1380
1381                         srcu_notifier_call_chain(
1382                                         &cpufreq_transition_notifier_list,
1383                                         CPUFREQ_RESUMECHANGE, &freqs);
1384                         adjust_jiffies(CPUFREQ_RESUMECHANGE, &freqs);
1385
1386                         cpu_policy->cur = cur_freq;
1387                 }
1388         }
1389
1390 out:
1391         schedule_work(&cpu_policy->update);
1392 fail:
1393         cpufreq_cpu_put(cpu_policy);
1394         return ret;
1395 }
1396
1397 static struct sysdev_driver cpufreq_sysdev_driver = {
1398         .add            = cpufreq_add_dev,
1399         .remove         = cpufreq_remove_dev,
1400         .suspend        = cpufreq_suspend,
1401         .resume         = cpufreq_resume,
1402 };
1403
1404
1405 /*********************************************************************
1406  *                     NOTIFIER LISTS INTERFACE                      *
1407  *********************************************************************/
1408
1409 /**
1410  *      cpufreq_register_notifier - register a driver with cpufreq
1411  *      @nb: notifier function to register
1412  *      @list: CPUFREQ_TRANSITION_NOTIFIER or CPUFREQ_POLICY_NOTIFIER
1413  *
1414  *      Add a driver to one of two lists: either a list of drivers that
1415  *      are notified about clock rate changes (once before and once after
1416  *      the transition), or a list of drivers that are notified about
1417  *      changes in cpufreq policy.
1418  *
1419  *      This function may sleep, and has the same return conditions as
1420  *      blocking_notifier_chain_register.
1421  */
1422 int cpufreq_register_notifier(struct notifier_block *nb, unsigned int list)
1423 {
1424         int ret;
1425
1426         WARN_ON(!init_cpufreq_transition_notifier_list_called);
1427
1428         switch (list) {
1429         case CPUFREQ_TRANSITION_NOTIFIER:
1430                 ret = srcu_notifier_chain_register(
1431                                 &cpufreq_transition_notifier_list, nb);
1432                 break;
1433         case CPUFREQ_POLICY_NOTIFIER:
1434                 ret = blocking_notifier_chain_register(
1435                                 &cpufreq_policy_notifier_list, nb);
1436                 break;
1437         default:
1438                 ret = -EINVAL;
1439         }
1440
1441         return ret;
1442 }
1443 EXPORT_SYMBOL(cpufreq_register_notifier);
1444
1445
1446 /**
1447  *      cpufreq_unregister_notifier - unregister a driver with cpufreq
1448  *      @nb: notifier block to be unregistered
1449  *      @list: CPUFREQ_TRANSITION_NOTIFIER or CPUFREQ_POLICY_NOTIFIER
1450  *
1451  *      Remove a driver from the CPU frequency notifier list.
1452  *
1453  *      This function may sleep, and has the same return conditions as
1454  *      blocking_notifier_chain_unregister.
1455  */
1456 int cpufreq_unregister_notifier(struct notifier_block *nb, unsigned int list)
1457 {
1458         int ret;
1459
1460         switch (list) {
1461         case CPUFREQ_TRANSITION_NOTIFIER:
1462                 ret = srcu_notifier_chain_unregister(
1463                                 &cpufreq_transition_notifier_list, nb);
1464                 break;
1465         case CPUFREQ_POLICY_NOTIFIER:
1466                 ret = blocking_notifier_chain_unregister(
1467                                 &cpufreq_policy_notifier_list, nb);
1468                 break;
1469         default:
1470                 ret = -EINVAL;
1471         }
1472
1473         return ret;
1474 }
1475 EXPORT_SYMBOL(cpufreq_unregister_notifier);
1476
1477
1478 /*********************************************************************
1479  *                              GOVERNORS                            *
1480  *********************************************************************/
1481
1482
1483 int __cpufreq_driver_target(struct cpufreq_policy *policy,
1484                             unsigned int target_freq,
1485                             unsigned int relation)
1486 {
1487         int retval = -EINVAL;
1488
1489         dprintk("target for CPU %u: %u kHz, relation %u\n", policy->cpu,
1490                 target_freq, relation);
1491         if (cpu_online(policy->cpu) && cpufreq_driver->target)
1492                 retval = cpufreq_driver->target(policy, target_freq, relation);
1493
1494         return retval;
1495 }
1496 EXPORT_SYMBOL_GPL(__cpufreq_driver_target);
1497
1498 int cpufreq_driver_target(struct cpufreq_policy *policy,
1499                           unsigned int target_freq,
1500                           unsigned int relation)
1501 {
1502         int ret = -EINVAL;
1503
1504         policy = cpufreq_cpu_get(policy->cpu);
1505         if (!policy)
1506                 goto no_policy;
1507
1508         if (unlikely(lock_policy_rwsem_write(policy->cpu)))
1509                 goto fail;
1510
1511         ret = __cpufreq_driver_target(policy, target_freq, relation);
1512
1513         unlock_policy_rwsem_write(policy->cpu);
1514
1515 fail:
1516         cpufreq_cpu_put(policy);
1517 no_policy:
1518         return ret;
1519 }
1520 EXPORT_SYMBOL_GPL(cpufreq_driver_target);
1521
1522 int __cpufreq_driver_getavg(struct cpufreq_policy *policy, unsigned int cpu)
1523 {
1524         int ret = 0;
1525
1526         policy = cpufreq_cpu_get(policy->cpu);
1527         if (!policy)
1528                 return -EINVAL;
1529
1530         if (cpu_online(cpu) && cpufreq_driver->getavg)
1531                 ret = cpufreq_driver->getavg(policy, cpu);
1532
1533         cpufreq_cpu_put(policy);
1534         return ret;
1535 }
1536 EXPORT_SYMBOL_GPL(__cpufreq_driver_getavg);
1537
1538 /*
1539  * when "event" is CPUFREQ_GOV_LIMITS
1540  */
1541
1542 static int __cpufreq_governor(struct cpufreq_policy *policy,
1543                                         unsigned int event)
1544 {
1545         int ret;
1546
1547         /* Only must be defined when default governor is known to have latency
1548            restrictions, like e.g. conservative or ondemand.
1549            That this is the case is already ensured in Kconfig
1550         */
1551 #ifdef CONFIG_CPU_FREQ_GOV_PERFORMANCE
1552         struct cpufreq_governor *gov = &cpufreq_gov_performance;
1553 #else
1554         struct cpufreq_governor *gov = NULL;
1555 #endif
1556
1557         if (policy->governor->max_transition_latency &&
1558             policy->cpuinfo.transition_latency >
1559             policy->governor->max_transition_latency) {
1560                 if (!gov)
1561                         return -EINVAL;
1562                 else {
1563                         printk(KERN_WARNING "%s governor failed, too long"
1564                                " transition latency of HW, fallback"
1565                                " to %s governor\n",
1566                                policy->governor->name,
1567                                gov->name);
1568                         policy->governor = gov;
1569                 }
1570         }
1571
1572         if (!try_module_get(policy->governor->owner))
1573                 return -EINVAL;
1574
1575         dprintk("__cpufreq_governor for CPU %u, event %u\n",
1576                                                 policy->cpu, event);
1577         ret = policy->governor->governor(policy, event);
1578
1579         /* we keep one module reference alive for
1580                         each CPU governed by this CPU */
1581         if ((event != CPUFREQ_GOV_START) || ret)
1582                 module_put(policy->governor->owner);
1583         if ((event == CPUFREQ_GOV_STOP) && !ret)
1584                 module_put(policy->governor->owner);
1585
1586         return ret;
1587 }
1588
1589
1590 int cpufreq_register_governor(struct cpufreq_governor *governor)
1591 {
1592         int err;
1593
1594         if (!governor)
1595                 return -EINVAL;
1596
1597         mutex_lock(&cpufreq_governor_mutex);
1598
1599         err = -EBUSY;
1600         if (__find_governor(governor->name) == NULL) {
1601                 err = 0;
1602                 list_add(&governor->governor_list, &cpufreq_governor_list);
1603         }
1604
1605         mutex_unlock(&cpufreq_governor_mutex);
1606         return err;
1607 }
1608 EXPORT_SYMBOL_GPL(cpufreq_register_governor);
1609
1610
1611 void cpufreq_unregister_governor(struct cpufreq_governor *governor)
1612 {
1613         if (!governor)
1614                 return;
1615
1616         mutex_lock(&cpufreq_governor_mutex);
1617         list_del(&governor->governor_list);
1618         mutex_unlock(&cpufreq_governor_mutex);
1619         return;
1620 }
1621 EXPORT_SYMBOL_GPL(cpufreq_unregister_governor);
1622
1623
1624
1625 /*********************************************************************
1626  *                          POLICY INTERFACE                         *
1627  *********************************************************************/
1628
1629 /**
1630  * cpufreq_get_policy - get the current cpufreq_policy
1631  * @policy: struct cpufreq_policy into which the current cpufreq_policy
1632  *      is written
1633  *
1634  * Reads the current cpufreq policy.
1635  */
1636 int cpufreq_get_policy(struct cpufreq_policy *policy, unsigned int cpu)
1637 {
1638         struct cpufreq_policy *cpu_policy;
1639         if (!policy)
1640                 return -EINVAL;
1641
1642         cpu_policy = cpufreq_cpu_get(cpu);
1643         if (!cpu_policy)
1644                 return -EINVAL;
1645
1646         memcpy(policy, cpu_policy, sizeof(struct cpufreq_policy));
1647
1648         cpufreq_cpu_put(cpu_policy);
1649         return 0;
1650 }
1651 EXPORT_SYMBOL(cpufreq_get_policy);
1652
1653
1654 /*
1655  * data   : current policy.
1656  * policy : policy to be set.
1657  */
1658 static int __cpufreq_set_policy(struct cpufreq_policy *data,
1659                                 struct cpufreq_policy *policy)
1660 {
1661         int ret = 0;
1662
1663         cpufreq_debug_disable_ratelimit();
1664         dprintk("setting new policy for CPU %u: %u - %u kHz\n", policy->cpu,
1665                 policy->min, policy->max);
1666
1667         memcpy(&policy->cpuinfo, &data->cpuinfo,
1668                                 sizeof(struct cpufreq_cpuinfo));
1669
1670         if (policy->min > data->max || policy->max < data->min) {
1671                 ret = -EINVAL;
1672                 goto error_out;
1673         }
1674
1675         /* verify the cpu speed can be set within this limit */
1676         ret = cpufreq_driver->verify(policy);
1677         if (ret)
1678                 goto error_out;
1679
1680         /* adjust if necessary - all reasons */
1681         blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
1682                         CPUFREQ_ADJUST, policy);
1683
1684         /* adjust if necessary - hardware incompatibility*/
1685         blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
1686                         CPUFREQ_INCOMPATIBLE, policy);
1687
1688         /* verify the cpu speed can be set within this limit,
1689            which might be different to the first one */
1690         ret = cpufreq_driver->verify(policy);
1691         if (ret)
1692                 goto error_out;
1693
1694         /* notification of the new policy */
1695         blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
1696                         CPUFREQ_NOTIFY, policy);
1697
1698         data->min = policy->min;
1699         data->max = policy->max;
1700
1701         dprintk("new min and max freqs are %u - %u kHz\n",
1702                                         data->min, data->max);
1703
1704         if (cpufreq_driver->setpolicy) {
1705                 data->policy = policy->policy;
1706                 dprintk("setting range\n");
1707                 ret = cpufreq_driver->setpolicy(policy);
1708         } else {
1709                 if (policy->governor != data->governor) {
1710                         /* save old, working values */
1711                         struct cpufreq_governor *old_gov = data->governor;
1712
1713                         dprintk("governor switch\n");
1714
1715                         /* end old governor */
1716                         if (data->governor)
1717                                 __cpufreq_governor(data, CPUFREQ_GOV_STOP);
1718
1719                         /* start new governor */
1720                         data->governor = policy->governor;
1721                         if (__cpufreq_governor(data, CPUFREQ_GOV_START)) {
1722                                 /* new governor failed, so re-start old one */
1723                                 dprintk("starting governor %s failed\n",
1724                                                         data->governor->name);
1725                                 if (old_gov) {
1726                                         data->governor = old_gov;
1727                                         __cpufreq_governor(data,
1728                                                            CPUFREQ_GOV_START);
1729                                 }
1730                                 ret = -EINVAL;
1731                                 goto error_out;
1732                         }
1733                         /* might be a policy change, too, so fall through */
1734                 }
1735                 dprintk("governor: change or update limits\n");
1736                 __cpufreq_governor(data, CPUFREQ_GOV_LIMITS);
1737         }
1738
1739 error_out:
1740         cpufreq_debug_enable_ratelimit();
1741         return ret;
1742 }
1743
1744 /**
1745  *      cpufreq_update_policy - re-evaluate an existing cpufreq policy
1746  *      @cpu: CPU which shall be re-evaluated
1747  *
1748  *      Usefull for policy notifiers which have different necessities
1749  *      at different times.
1750  */
1751 int cpufreq_update_policy(unsigned int cpu)
1752 {
1753         struct cpufreq_policy *data = cpufreq_cpu_get(cpu);
1754         struct cpufreq_policy policy;
1755         int ret;
1756
1757         if (!data) {
1758                 ret = -ENODEV;
1759                 goto no_policy;
1760         }
1761
1762         if (unlikely(lock_policy_rwsem_write(cpu))) {
1763                 ret = -EINVAL;
1764                 goto fail;
1765         }
1766
1767         dprintk("updating policy for CPU %u\n", cpu);
1768         memcpy(&policy, data, sizeof(struct cpufreq_policy));
1769         policy.min = data->user_policy.min;
1770         policy.max = data->user_policy.max;
1771         policy.policy = data->user_policy.policy;
1772         policy.governor = data->user_policy.governor;
1773
1774         /* BIOS might change freq behind our back
1775           -> ask driver for current freq and notify governors about a change */
1776         if (cpufreq_driver->get) {
1777                 policy.cur = cpufreq_driver->get(cpu);
1778                 if (!data->cur) {
1779                         dprintk("Driver did not initialize current freq");
1780                         data->cur = policy.cur;
1781                 } else {
1782                         if (data->cur != policy.cur)
1783                                 cpufreq_out_of_sync(cpu, data->cur,
1784                                                                 policy.cur);
1785                 }
1786         }
1787
1788         ret = __cpufreq_set_policy(data, &policy);
1789
1790         unlock_policy_rwsem_write(cpu);
1791
1792 fail:
1793         cpufreq_cpu_put(data);
1794 no_policy:
1795         return ret;
1796 }
1797 EXPORT_SYMBOL(cpufreq_update_policy);
1798
1799 static int __cpuinit cpufreq_cpu_callback(struct notifier_block *nfb,
1800                                         unsigned long action, void *hcpu)
1801 {
1802         unsigned int cpu = (unsigned long)hcpu;
1803         struct sys_device *sys_dev;
1804
1805         sys_dev = get_cpu_sysdev(cpu);
1806         if (sys_dev) {
1807                 switch (action) {
1808                 case CPU_ONLINE:
1809                 case CPU_ONLINE_FROZEN:
1810                         cpufreq_add_dev(sys_dev);
1811                         break;
1812                 case CPU_DOWN_PREPARE:
1813                 case CPU_DOWN_PREPARE_FROZEN:
1814                         if (unlikely(lock_policy_rwsem_write(cpu)))
1815                                 BUG();
1816
1817                         __cpufreq_remove_dev(sys_dev);
1818                         break;
1819                 case CPU_DOWN_FAILED:
1820                 case CPU_DOWN_FAILED_FROZEN:
1821                         cpufreq_add_dev(sys_dev);
1822                         break;
1823                 }
1824         }
1825         return NOTIFY_OK;
1826 }
1827
1828 static struct notifier_block __refdata cpufreq_cpu_notifier =
1829 {
1830     .notifier_call = cpufreq_cpu_callback,
1831 };
1832
1833 /*********************************************************************
1834  *               REGISTER / UNREGISTER CPUFREQ DRIVER                *
1835  *********************************************************************/
1836
1837 /**
1838  * cpufreq_register_driver - register a CPU Frequency driver
1839  * @driver_data: A struct cpufreq_driver containing the values#
1840  * submitted by the CPU Frequency driver.
1841  *
1842  *   Registers a CPU Frequency driver to this core code. This code
1843  * returns zero on success, -EBUSY when another driver got here first
1844  * (and isn't unregistered in the meantime).
1845  *
1846  */
1847 int cpufreq_register_driver(struct cpufreq_driver *driver_data)
1848 {
1849         unsigned long flags;
1850         int ret;
1851
1852         if (!driver_data || !driver_data->verify || !driver_data->init ||
1853             ((!driver_data->setpolicy) && (!driver_data->target)))
1854                 return -EINVAL;
1855
1856         dprintk("trying to register driver %s\n", driver_data->name);
1857
1858         if (driver_data->setpolicy)
1859                 driver_data->flags |= CPUFREQ_CONST_LOOPS;
1860
1861         spin_lock_irqsave(&cpufreq_driver_lock, flags);
1862         if (cpufreq_driver) {
1863                 spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
1864                 return -EBUSY;
1865         }
1866         cpufreq_driver = driver_data;
1867         spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
1868
1869         ret = sysdev_driver_register(&cpu_sysdev_class,
1870                                         &cpufreq_sysdev_driver);
1871
1872         if ((!ret) && !(cpufreq_driver->flags & CPUFREQ_STICKY)) {
1873                 int i;
1874                 ret = -ENODEV;
1875
1876                 /* check for at least one working CPU */
1877                 for (i = 0; i < nr_cpu_ids; i++)
1878                         if (cpu_possible(i) && per_cpu(cpufreq_cpu_data, i)) {
1879                                 ret = 0;
1880                                 break;
1881                         }
1882
1883                 /* if all ->init() calls failed, unregister */
1884                 if (ret) {
1885                         dprintk("no CPU initialized for driver %s\n",
1886                                                         driver_data->name);
1887                         sysdev_driver_unregister(&cpu_sysdev_class,
1888                                                 &cpufreq_sysdev_driver);
1889
1890                         spin_lock_irqsave(&cpufreq_driver_lock, flags);
1891                         cpufreq_driver = NULL;
1892                         spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
1893                 }
1894         }
1895
1896         if (!ret) {
1897                 register_hotcpu_notifier(&cpufreq_cpu_notifier);
1898                 dprintk("driver %s up and running\n", driver_data->name);
1899                 cpufreq_debug_enable_ratelimit();
1900         }
1901
1902         return ret;
1903 }
1904 EXPORT_SYMBOL_GPL(cpufreq_register_driver);
1905
1906
1907 /**
1908  * cpufreq_unregister_driver - unregister the current CPUFreq driver
1909  *
1910  *    Unregister the current CPUFreq driver. Only call this if you have
1911  * the right to do so, i.e. if you have succeeded in initialising before!
1912  * Returns zero if successful, and -EINVAL if the cpufreq_driver is
1913  * currently not initialised.
1914  */
1915 int cpufreq_unregister_driver(struct cpufreq_driver *driver)
1916 {
1917         unsigned long flags;
1918
1919         cpufreq_debug_disable_ratelimit();
1920
1921         if (!cpufreq_driver || (driver != cpufreq_driver)) {
1922                 cpufreq_debug_enable_ratelimit();
1923                 return -EINVAL;
1924         }
1925
1926         dprintk("unregistering driver %s\n", driver->name);
1927
1928         sysdev_driver_unregister(&cpu_sysdev_class, &cpufreq_sysdev_driver);
1929         unregister_hotcpu_notifier(&cpufreq_cpu_notifier);
1930
1931         spin_lock_irqsave(&cpufreq_driver_lock, flags);
1932         cpufreq_driver = NULL;
1933         spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
1934
1935         return 0;
1936 }
1937 EXPORT_SYMBOL_GPL(cpufreq_unregister_driver);
1938
1939 static int __init cpufreq_core_init(void)
1940 {
1941         int cpu;
1942
1943         for_each_possible_cpu(cpu) {
1944                 per_cpu(policy_cpu, cpu) = -1;
1945                 init_rwsem(&per_cpu(cpu_policy_rwsem, cpu));
1946         }
1947         return 0;
1948 }
1949
1950 core_initcall(cpufreq_core_init);