Merge branch 'master' of /home/trondmy/kernel/linux-2.6/
[linux-2.6] / drivers / cpufreq / cpufreq_ondemand.c
1 /*
2  *  drivers/cpufreq/cpufreq_ondemand.c
3  *
4  *  Copyright (C)  2001 Russell King
5  *            (C)  2003 Venkatesh Pallipadi <venkatesh.pallipadi@intel.com>.
6  *                      Jun Nakajima <jun.nakajima@intel.com>
7  *
8  * This program is free software; you can redistribute it and/or modify
9  * it under the terms of the GNU General Public License version 2 as
10  * published by the Free Software Foundation.
11  */
12
13 #include <linux/kernel.h>
14 #include <linux/module.h>
15 #include <linux/smp.h>
16 #include <linux/init.h>
17 #include <linux/interrupt.h>
18 #include <linux/ctype.h>
19 #include <linux/cpufreq.h>
20 #include <linux/sysctl.h>
21 #include <linux/types.h>
22 #include <linux/fs.h>
23 #include <linux/sysfs.h>
24 #include <linux/cpu.h>
25 #include <linux/sched.h>
26 #include <linux/kmod.h>
27 #include <linux/workqueue.h>
28 #include <linux/jiffies.h>
29 #include <linux/kernel_stat.h>
30 #include <linux/percpu.h>
31 #include <linux/mutex.h>
32
33 /*
34  * dbs is used in this file as a shortform for demandbased switching
35  * It helps to keep variable names smaller, simpler
36  */
37
38 #define DEF_FREQUENCY_UP_THRESHOLD              (80)
39 #define MIN_FREQUENCY_UP_THRESHOLD              (11)
40 #define MAX_FREQUENCY_UP_THRESHOLD              (100)
41
42 /*
43  * The polling frequency of this governor depends on the capability of
44  * the processor. Default polling frequency is 1000 times the transition
45  * latency of the processor. The governor will work on any processor with
46  * transition latency <= 10mS, using appropriate sampling
47  * rate.
48  * For CPUs with transition latency > 10mS (mostly drivers with CPUFREQ_ETERNAL)
49  * this governor will not work.
50  * All times here are in uS.
51  */
52 static unsigned int def_sampling_rate;
53 #define MIN_SAMPLING_RATE_RATIO                 (2)
54 /* for correct statistics, we need at least 10 ticks between each measure */
55 #define MIN_STAT_SAMPLING_RATE                  (MIN_SAMPLING_RATE_RATIO * jiffies_to_usecs(10))
56 #define MIN_SAMPLING_RATE                       (def_sampling_rate / MIN_SAMPLING_RATE_RATIO)
57 #define MAX_SAMPLING_RATE                       (500 * def_sampling_rate)
58 #define DEF_SAMPLING_RATE_LATENCY_MULTIPLIER    (1000)
59 #define DEF_SAMPLING_DOWN_FACTOR                (1)
60 #define MAX_SAMPLING_DOWN_FACTOR                (10)
61 #define TRANSITION_LATENCY_LIMIT                (10 * 1000)
62
63 static void do_dbs_timer(void *data);
64
65 struct cpu_dbs_info_s {
66         struct cpufreq_policy *cur_policy;
67         unsigned int prev_cpu_idle_up;
68         unsigned int prev_cpu_idle_down;
69         unsigned int enable;
70 };
71 static DEFINE_PER_CPU(struct cpu_dbs_info_s, cpu_dbs_info);
72
73 static unsigned int dbs_enable; /* number of CPUs using this policy */
74
75 /*
76  * DEADLOCK ALERT! There is a ordering requirement between cpu_hotplug
77  * lock and dbs_mutex. cpu_hotplug lock should always be held before
78  * dbs_mutex. If any function that can potentially take cpu_hotplug lock
79  * (like __cpufreq_driver_target()) is being called with dbs_mutex taken, then
80  * cpu_hotplug lock should be taken before that. Note that cpu_hotplug lock
81  * is recursive for the same process. -Venki
82  */
83 static DEFINE_MUTEX (dbs_mutex);
84 static DECLARE_WORK     (dbs_work, do_dbs_timer, NULL);
85
86 static struct workqueue_struct *dbs_workq;
87
88 struct dbs_tuners {
89         unsigned int sampling_rate;
90         unsigned int sampling_down_factor;
91         unsigned int up_threshold;
92         unsigned int ignore_nice;
93 };
94
95 static struct dbs_tuners dbs_tuners_ins = {
96         .up_threshold = DEF_FREQUENCY_UP_THRESHOLD,
97         .sampling_down_factor = DEF_SAMPLING_DOWN_FACTOR,
98         .ignore_nice = 0,
99 };
100
101 static inline unsigned int get_cpu_idle_time(unsigned int cpu)
102 {
103         return  kstat_cpu(cpu).cpustat.idle +
104                 kstat_cpu(cpu).cpustat.iowait +
105                 ( dbs_tuners_ins.ignore_nice ?
106                   kstat_cpu(cpu).cpustat.nice :
107                   0);
108 }
109
110 /************************** sysfs interface ************************/
111 static ssize_t show_sampling_rate_max(struct cpufreq_policy *policy, char *buf)
112 {
113         return sprintf (buf, "%u\n", MAX_SAMPLING_RATE);
114 }
115
116 static ssize_t show_sampling_rate_min(struct cpufreq_policy *policy, char *buf)
117 {
118         return sprintf (buf, "%u\n", MIN_SAMPLING_RATE);
119 }
120
121 #define define_one_ro(_name)            \
122 static struct freq_attr _name =         \
123 __ATTR(_name, 0444, show_##_name, NULL)
124
125 define_one_ro(sampling_rate_max);
126 define_one_ro(sampling_rate_min);
127
128 /* cpufreq_ondemand Governor Tunables */
129 #define show_one(file_name, object)                                     \
130 static ssize_t show_##file_name                                         \
131 (struct cpufreq_policy *unused, char *buf)                              \
132 {                                                                       \
133         return sprintf(buf, "%u\n", dbs_tuners_ins.object);             \
134 }
135 show_one(sampling_rate, sampling_rate);
136 show_one(sampling_down_factor, sampling_down_factor);
137 show_one(up_threshold, up_threshold);
138 show_one(ignore_nice_load, ignore_nice);
139
140 static ssize_t store_sampling_down_factor(struct cpufreq_policy *unused,
141                 const char *buf, size_t count)
142 {
143         unsigned int input;
144         int ret;
145         ret = sscanf (buf, "%u", &input);
146         if (ret != 1 )
147                 return -EINVAL;
148
149         if (input > MAX_SAMPLING_DOWN_FACTOR || input < 1)
150                 return -EINVAL;
151
152         mutex_lock(&dbs_mutex);
153         dbs_tuners_ins.sampling_down_factor = input;
154         mutex_unlock(&dbs_mutex);
155
156         return count;
157 }
158
159 static ssize_t store_sampling_rate(struct cpufreq_policy *unused,
160                 const char *buf, size_t count)
161 {
162         unsigned int input;
163         int ret;
164         ret = sscanf (buf, "%u", &input);
165
166         mutex_lock(&dbs_mutex);
167         if (ret != 1 || input > MAX_SAMPLING_RATE || input < MIN_SAMPLING_RATE) {
168                 mutex_unlock(&dbs_mutex);
169                 return -EINVAL;
170         }
171
172         dbs_tuners_ins.sampling_rate = input;
173         mutex_unlock(&dbs_mutex);
174
175         return count;
176 }
177
178 static ssize_t store_up_threshold(struct cpufreq_policy *unused,
179                 const char *buf, size_t count)
180 {
181         unsigned int input;
182         int ret;
183         ret = sscanf (buf, "%u", &input);
184
185         mutex_lock(&dbs_mutex);
186         if (ret != 1 || input > MAX_FREQUENCY_UP_THRESHOLD ||
187                         input < MIN_FREQUENCY_UP_THRESHOLD) {
188                 mutex_unlock(&dbs_mutex);
189                 return -EINVAL;
190         }
191
192         dbs_tuners_ins.up_threshold = input;
193         mutex_unlock(&dbs_mutex);
194
195         return count;
196 }
197
198 static ssize_t store_ignore_nice_load(struct cpufreq_policy *policy,
199                 const char *buf, size_t count)
200 {
201         unsigned int input;
202         int ret;
203
204         unsigned int j;
205
206         ret = sscanf (buf, "%u", &input);
207         if ( ret != 1 )
208                 return -EINVAL;
209
210         if ( input > 1 )
211                 input = 1;
212
213         mutex_lock(&dbs_mutex);
214         if ( input == dbs_tuners_ins.ignore_nice ) { /* nothing to do */
215                 mutex_unlock(&dbs_mutex);
216                 return count;
217         }
218         dbs_tuners_ins.ignore_nice = input;
219
220         /* we need to re-evaluate prev_cpu_idle_up and prev_cpu_idle_down */
221         for_each_online_cpu(j) {
222                 struct cpu_dbs_info_s *j_dbs_info;
223                 j_dbs_info = &per_cpu(cpu_dbs_info, j);
224                 j_dbs_info->prev_cpu_idle_up = get_cpu_idle_time(j);
225                 j_dbs_info->prev_cpu_idle_down = j_dbs_info->prev_cpu_idle_up;
226         }
227         mutex_unlock(&dbs_mutex);
228
229         return count;
230 }
231
232 #define define_one_rw(_name) \
233 static struct freq_attr _name = \
234 __ATTR(_name, 0644, show_##_name, store_##_name)
235
236 define_one_rw(sampling_rate);
237 define_one_rw(sampling_down_factor);
238 define_one_rw(up_threshold);
239 define_one_rw(ignore_nice_load);
240
241 static struct attribute * dbs_attributes[] = {
242         &sampling_rate_max.attr,
243         &sampling_rate_min.attr,
244         &sampling_rate.attr,
245         &sampling_down_factor.attr,
246         &up_threshold.attr,
247         &ignore_nice_load.attr,
248         NULL
249 };
250
251 static struct attribute_group dbs_attr_group = {
252         .attrs = dbs_attributes,
253         .name = "ondemand",
254 };
255
256 /************************** sysfs end ************************/
257
258 static void dbs_check_cpu(int cpu)
259 {
260         unsigned int idle_ticks, up_idle_ticks, total_ticks;
261         unsigned int freq_next;
262         unsigned int freq_down_sampling_rate;
263         static int down_skip[NR_CPUS];
264         struct cpu_dbs_info_s *this_dbs_info;
265
266         struct cpufreq_policy *policy;
267         unsigned int j;
268
269         this_dbs_info = &per_cpu(cpu_dbs_info, cpu);
270         if (!this_dbs_info->enable)
271                 return;
272
273         policy = this_dbs_info->cur_policy;
274         /*
275          * Every sampling_rate, we check, if current idle time is less
276          * than 20% (default), then we try to increase frequency
277          * Every sampling_rate*sampling_down_factor, we look for a the lowest
278          * frequency which can sustain the load while keeping idle time over
279          * 30%. If such a frequency exist, we try to decrease to this frequency.
280          *
281          * Any frequency increase takes it to the maximum frequency.
282          * Frequency reduction happens at minimum steps of
283          * 5% (default) of current frequency
284          */
285
286         /* Check for frequency increase */
287         idle_ticks = UINT_MAX;
288         for_each_cpu_mask(j, policy->cpus) {
289                 unsigned int tmp_idle_ticks, total_idle_ticks;
290                 struct cpu_dbs_info_s *j_dbs_info;
291
292                 j_dbs_info = &per_cpu(cpu_dbs_info, j);
293                 total_idle_ticks = get_cpu_idle_time(j);
294                 tmp_idle_ticks = total_idle_ticks -
295                         j_dbs_info->prev_cpu_idle_up;
296                 j_dbs_info->prev_cpu_idle_up = total_idle_ticks;
297
298                 if (tmp_idle_ticks < idle_ticks)
299                         idle_ticks = tmp_idle_ticks;
300         }
301
302         /* Scale idle ticks by 100 and compare with up and down ticks */
303         idle_ticks *= 100;
304         up_idle_ticks = (100 - dbs_tuners_ins.up_threshold) *
305                         usecs_to_jiffies(dbs_tuners_ins.sampling_rate);
306
307         if (idle_ticks < up_idle_ticks) {
308                 down_skip[cpu] = 0;
309                 for_each_cpu_mask(j, policy->cpus) {
310                         struct cpu_dbs_info_s *j_dbs_info;
311
312                         j_dbs_info = &per_cpu(cpu_dbs_info, j);
313                         j_dbs_info->prev_cpu_idle_down =
314                                         j_dbs_info->prev_cpu_idle_up;
315                 }
316                 /* if we are already at full speed then break out early */
317                 if (policy->cur == policy->max)
318                         return;
319
320                 __cpufreq_driver_target(policy, policy->max,
321                         CPUFREQ_RELATION_H);
322                 return;
323         }
324
325         /* Check for frequency decrease */
326         down_skip[cpu]++;
327         if (down_skip[cpu] < dbs_tuners_ins.sampling_down_factor)
328                 return;
329
330         idle_ticks = UINT_MAX;
331         for_each_cpu_mask(j, policy->cpus) {
332                 unsigned int tmp_idle_ticks, total_idle_ticks;
333                 struct cpu_dbs_info_s *j_dbs_info;
334
335                 j_dbs_info = &per_cpu(cpu_dbs_info, j);
336                 /* Check for frequency decrease */
337                 total_idle_ticks = j_dbs_info->prev_cpu_idle_up;
338                 tmp_idle_ticks = total_idle_ticks -
339                         j_dbs_info->prev_cpu_idle_down;
340                 j_dbs_info->prev_cpu_idle_down = total_idle_ticks;
341
342                 if (tmp_idle_ticks < idle_ticks)
343                         idle_ticks = tmp_idle_ticks;
344         }
345
346         down_skip[cpu] = 0;
347         /* if we cannot reduce the frequency anymore, break out early */
348         if (policy->cur == policy->min)
349                 return;
350
351         /* Compute how many ticks there are between two measurements */
352         freq_down_sampling_rate = dbs_tuners_ins.sampling_rate *
353                 dbs_tuners_ins.sampling_down_factor;
354         total_ticks = usecs_to_jiffies(freq_down_sampling_rate);
355
356         /*
357          * The optimal frequency is the frequency that is the lowest that
358          * can support the current CPU usage without triggering the up
359          * policy. To be safe, we focus 10 points under the threshold.
360          */
361         freq_next = ((total_ticks - idle_ticks) * 100) / total_ticks;
362         freq_next = (freq_next * policy->cur) /
363                         (dbs_tuners_ins.up_threshold - 10);
364
365         if (freq_next < policy->min)
366                 freq_next = policy->min;
367
368         if (freq_next <= ((policy->cur * 95) / 100))
369                 __cpufreq_driver_target(policy, freq_next, CPUFREQ_RELATION_L);
370 }
371
372 static void do_dbs_timer(void *data)
373 {
374         int i;
375         lock_cpu_hotplug();
376         mutex_lock(&dbs_mutex);
377         for_each_online_cpu(i)
378                 dbs_check_cpu(i);
379         queue_delayed_work(dbs_workq, &dbs_work,
380                            usecs_to_jiffies(dbs_tuners_ins.sampling_rate));
381         mutex_unlock(&dbs_mutex);
382         unlock_cpu_hotplug();
383 }
384
385 static inline void dbs_timer_init(void)
386 {
387         INIT_WORK(&dbs_work, do_dbs_timer, NULL);
388         if (!dbs_workq)
389                 dbs_workq = create_singlethread_workqueue("ondemand");
390         if (!dbs_workq) {
391                 printk(KERN_ERR "ondemand: Cannot initialize kernel thread\n");
392                 return;
393         }
394         queue_delayed_work(dbs_workq, &dbs_work,
395                            usecs_to_jiffies(dbs_tuners_ins.sampling_rate));
396         return;
397 }
398
399 static inline void dbs_timer_exit(void)
400 {
401         if (dbs_workq)
402                 cancel_rearming_delayed_workqueue(dbs_workq, &dbs_work);
403 }
404
405 static int cpufreq_governor_dbs(struct cpufreq_policy *policy,
406                                    unsigned int event)
407 {
408         unsigned int cpu = policy->cpu;
409         struct cpu_dbs_info_s *this_dbs_info;
410         unsigned int j;
411
412         this_dbs_info = &per_cpu(cpu_dbs_info, cpu);
413
414         switch (event) {
415         case CPUFREQ_GOV_START:
416                 if ((!cpu_online(cpu)) ||
417                     (!policy->cur))
418                         return -EINVAL;
419
420                 if (policy->cpuinfo.transition_latency >
421                                 (TRANSITION_LATENCY_LIMIT * 1000)) {
422                         printk(KERN_WARNING "ondemand governor failed to load "
423                                "due to too long transition latency\n");
424                         return -EINVAL;
425                 }
426                 if (this_dbs_info->enable) /* Already enabled */
427                         break;
428
429                 mutex_lock(&dbs_mutex);
430                 for_each_cpu_mask(j, policy->cpus) {
431                         struct cpu_dbs_info_s *j_dbs_info;
432                         j_dbs_info = &per_cpu(cpu_dbs_info, j);
433                         j_dbs_info->cur_policy = policy;
434
435                         j_dbs_info->prev_cpu_idle_up = get_cpu_idle_time(j);
436                         j_dbs_info->prev_cpu_idle_down
437                                 = j_dbs_info->prev_cpu_idle_up;
438                 }
439                 this_dbs_info->enable = 1;
440                 sysfs_create_group(&policy->kobj, &dbs_attr_group);
441                 dbs_enable++;
442                 /*
443                  * Start the timerschedule work, when this governor
444                  * is used for first time
445                  */
446                 if (dbs_enable == 1) {
447                         unsigned int latency;
448                         /* policy latency is in nS. Convert it to uS first */
449                         latency = policy->cpuinfo.transition_latency / 1000;
450                         if (latency == 0)
451                                 latency = 1;
452
453                         def_sampling_rate = latency *
454                                         DEF_SAMPLING_RATE_LATENCY_MULTIPLIER;
455
456                         if (def_sampling_rate < MIN_STAT_SAMPLING_RATE)
457                                 def_sampling_rate = MIN_STAT_SAMPLING_RATE;
458
459                         dbs_tuners_ins.sampling_rate = def_sampling_rate;
460                         dbs_timer_init();
461                 }
462
463                 mutex_unlock(&dbs_mutex);
464                 break;
465
466         case CPUFREQ_GOV_STOP:
467                 mutex_lock(&dbs_mutex);
468                 this_dbs_info->enable = 0;
469                 sysfs_remove_group(&policy->kobj, &dbs_attr_group);
470                 dbs_enable--;
471                 /*
472                  * Stop the timerschedule work, when this governor
473                  * is used for first time
474                  */
475                 if (dbs_enable == 0)
476                         dbs_timer_exit();
477
478                 mutex_unlock(&dbs_mutex);
479
480                 break;
481
482         case CPUFREQ_GOV_LIMITS:
483                 lock_cpu_hotplug();
484                 mutex_lock(&dbs_mutex);
485                 if (policy->max < this_dbs_info->cur_policy->cur)
486                         __cpufreq_driver_target(
487                                         this_dbs_info->cur_policy,
488                                         policy->max, CPUFREQ_RELATION_H);
489                 else if (policy->min > this_dbs_info->cur_policy->cur)
490                         __cpufreq_driver_target(
491                                         this_dbs_info->cur_policy,
492                                         policy->min, CPUFREQ_RELATION_L);
493                 mutex_unlock(&dbs_mutex);
494                 unlock_cpu_hotplug();
495                 break;
496         }
497         return 0;
498 }
499
500 static struct cpufreq_governor cpufreq_gov_dbs = {
501         .name           = "ondemand",
502         .governor       = cpufreq_governor_dbs,
503         .owner          = THIS_MODULE,
504 };
505
506 static int __init cpufreq_gov_dbs_init(void)
507 {
508         return cpufreq_register_governor(&cpufreq_gov_dbs);
509 }
510
511 static void __exit cpufreq_gov_dbs_exit(void)
512 {
513         /* Make sure that the scheduled work is indeed not running.
514            Assumes the timer has been cancelled first. */
515         if (dbs_workq) {
516                 flush_workqueue(dbs_workq);
517                 destroy_workqueue(dbs_workq);
518         }
519
520         cpufreq_unregister_governor(&cpufreq_gov_dbs);
521 }
522
523
524 MODULE_AUTHOR ("Venkatesh Pallipadi <venkatesh.pallipadi@intel.com>");
525 MODULE_DESCRIPTION ("'cpufreq_ondemand' - A dynamic cpufreq governor for "
526                 "Low Latency Frequency Transition capable processors");
527 MODULE_LICENSE ("GPL");
528
529 module_init(cpufreq_gov_dbs_init);
530 module_exit(cpufreq_gov_dbs_exit);