diff options
Diffstat (limited to 'drivers/cpufreq/cpufreq_ondemand.c')
| -rw-r--r-- | drivers/cpufreq/cpufreq_ondemand.c | 363 |
1 files changed, 234 insertions, 129 deletions
diff --git a/drivers/cpufreq/cpufreq_ondemand.c b/drivers/cpufreq/cpufreq_ondemand.c index f3eb26cd848..b0ffef96bf7 100644 --- a/drivers/cpufreq/cpufreq_ondemand.c +++ b/drivers/cpufreq/cpufreq_ondemand.c @@ -20,9 +20,11 @@ #include <linux/module.h> #include <linux/mutex.h> #include <linux/percpu-defs.h> +#include <linux/slab.h> #include <linux/sysfs.h> #include <linux/tick.h> #include <linux/types.h> +#include <linux/cpu.h> #include "cpufreq_governor.h" @@ -37,22 +39,14 @@ #define MIN_FREQUENCY_UP_THRESHOLD (11) #define MAX_FREQUENCY_UP_THRESHOLD (100) -static struct dbs_data od_dbs_data; static DEFINE_PER_CPU(struct od_cpu_dbs_info_s, od_cpu_dbs_info); +static struct od_ops od_ops; + #ifndef CONFIG_CPU_FREQ_DEFAULT_GOV_ONDEMAND static struct cpufreq_governor cpufreq_gov_ondemand; #endif -static struct od_dbs_tuners od_tuners = { - .up_threshold = DEF_FREQUENCY_UP_THRESHOLD, - .sampling_down_factor = DEF_SAMPLING_DOWN_FACTOR, - .adj_up_threshold = DEF_FREQUENCY_UP_THRESHOLD - - DEF_FREQUENCY_DOWN_DIFFERENTIAL, - .ignore_nice = 0, - .powersave_bias = 0, -}; - static void ondemand_powersave_bias_init_cpu(int cpu) { struct od_cpu_dbs_info_s *dbs_info = &per_cpu(od_cpu_dbs_info, cpu); @@ -89,7 +83,7 @@ static int should_io_be_busy(void) * Returns the freq_hi to be used right now and will set freq_hi_jiffies, * freq_lo, and freq_lo_jiffies in percpu area for averaging freqs. */ -static unsigned int powersave_bias_target(struct cpufreq_policy *policy, +static unsigned int generic_powersave_bias_target(struct cpufreq_policy *policy, unsigned int freq_next, unsigned int relation) { unsigned int freq_req, freq_reduc, freq_avg; @@ -98,6 +92,8 @@ static unsigned int powersave_bias_target(struct cpufreq_policy *policy, unsigned int jiffies_total, jiffies_hi, jiffies_lo; struct od_cpu_dbs_info_s *dbs_info = &per_cpu(od_cpu_dbs_info, policy->cpu); + struct dbs_data *dbs_data = policy->governor_data; + struct od_dbs_tuners *od_tuners = dbs_data->tuners; if (!dbs_info->freq_table) { dbs_info->freq_lo = 0; @@ -108,7 +104,7 @@ static unsigned int powersave_bias_target(struct cpufreq_policy *policy, cpufreq_frequency_table_target(policy, dbs_info->freq_table, freq_next, relation, &index); freq_req = dbs_info->freq_table[index].frequency; - freq_reduc = freq_req * od_tuners.powersave_bias / 1000; + freq_reduc = freq_req * od_tuners->powersave_bias / 1000; freq_avg = freq_req - freq_reduc; /* Find freq bounds for freq_avg in freq_table */ @@ -127,7 +123,7 @@ static unsigned int powersave_bias_target(struct cpufreq_policy *policy, dbs_info->freq_lo_jiffies = 0; return freq_lo; } - jiffies_total = usecs_to_jiffies(od_tuners.sampling_rate); + jiffies_total = usecs_to_jiffies(od_tuners->sampling_rate); jiffies_hi = (freq_avg - freq_lo) * jiffies_total; jiffies_hi += ((freq_hi - freq_lo) / 2); jiffies_hi /= (freq_hi - freq_lo); @@ -148,12 +144,16 @@ static void ondemand_powersave_bias_init(void) static void dbs_freq_increase(struct cpufreq_policy *p, unsigned int freq) { - if (od_tuners.powersave_bias) - freq = powersave_bias_target(p, freq, CPUFREQ_RELATION_H); + struct dbs_data *dbs_data = p->governor_data; + struct od_dbs_tuners *od_tuners = dbs_data->tuners; + + if (od_tuners->powersave_bias) + freq = od_ops.powersave_bias_target(p, freq, + CPUFREQ_RELATION_H); else if (p->cur == p->max) return; - __cpufreq_driver_target(p, freq, od_tuners.powersave_bias ? + __cpufreq_driver_target(p, freq, od_tuners->powersave_bias ? CPUFREQ_RELATION_L : CPUFREQ_RELATION_H); } @@ -170,15 +170,17 @@ static void od_check_cpu(int cpu, unsigned int load_freq) { struct od_cpu_dbs_info_s *dbs_info = &per_cpu(od_cpu_dbs_info, cpu); struct cpufreq_policy *policy = dbs_info->cdbs.cur_policy; + struct dbs_data *dbs_data = policy->governor_data; + struct od_dbs_tuners *od_tuners = dbs_data->tuners; dbs_info->freq_lo = 0; /* Check for frequency increase */ - if (load_freq > od_tuners.up_threshold * policy->cur) { + if (load_freq > od_tuners->up_threshold * policy->cur) { /* If switching to max speed, apply sampling_down_factor */ if (policy->cur < policy->max) dbs_info->rate_mult = - od_tuners.sampling_down_factor; + od_tuners->sampling_down_factor; dbs_freq_increase(policy, policy->max); return; } @@ -193,9 +195,10 @@ static void od_check_cpu(int cpu, unsigned int load_freq) * support the current CPU usage without triggering the up policy. To be * safe, we focus 10 points under the threshold. */ - if (load_freq < od_tuners.adj_up_threshold * policy->cur) { + if (load_freq < od_tuners->adj_up_threshold + * policy->cur) { unsigned int freq_next; - freq_next = load_freq / od_tuners.adj_up_threshold; + freq_next = load_freq / od_tuners->adj_up_threshold; /* No longer fully busy, reset rate_mult */ dbs_info->rate_mult = 1; @@ -203,65 +206,62 @@ static void od_check_cpu(int cpu, unsigned int load_freq) if (freq_next < policy->min) freq_next = policy->min; - if (!od_tuners.powersave_bias) { + if (!od_tuners->powersave_bias) { __cpufreq_driver_target(policy, freq_next, CPUFREQ_RELATION_L); - } else { - int freq = powersave_bias_target(policy, freq_next, - CPUFREQ_RELATION_L); - __cpufreq_driver_target(policy, freq, - CPUFREQ_RELATION_L); + return; } + + freq_next = od_ops.powersave_bias_target(policy, freq_next, + CPUFREQ_RELATION_L); + __cpufreq_driver_target(policy, freq_next, CPUFREQ_RELATION_L); } } static void od_dbs_timer(struct work_struct *work) { - struct delayed_work *dw = to_delayed_work(work); struct od_cpu_dbs_info_s *dbs_info = container_of(work, struct od_cpu_dbs_info_s, cdbs.work.work); unsigned int cpu = dbs_info->cdbs.cur_policy->cpu; struct od_cpu_dbs_info_s *core_dbs_info = &per_cpu(od_cpu_dbs_info, cpu); - int delay, sample_type = core_dbs_info->sample_type; - bool eval_load; + struct dbs_data *dbs_data = dbs_info->cdbs.cur_policy->governor_data; + struct od_dbs_tuners *od_tuners = dbs_data->tuners; + int delay = 0, sample_type = core_dbs_info->sample_type; + bool modify_all = true; mutex_lock(&core_dbs_info->cdbs.timer_mutex); - eval_load = need_load_eval(&core_dbs_info->cdbs, - od_tuners.sampling_rate); + if (!need_load_eval(&core_dbs_info->cdbs, od_tuners->sampling_rate)) { + modify_all = false; + goto max_delay; + } /* Common NORMAL_SAMPLE setup */ core_dbs_info->sample_type = OD_NORMAL_SAMPLE; if (sample_type == OD_SUB_SAMPLE) { delay = core_dbs_info->freq_lo_jiffies; - if (eval_load) - __cpufreq_driver_target(core_dbs_info->cdbs.cur_policy, - core_dbs_info->freq_lo, - CPUFREQ_RELATION_H); + __cpufreq_driver_target(core_dbs_info->cdbs.cur_policy, + core_dbs_info->freq_lo, CPUFREQ_RELATION_H); } else { - if (eval_load) - dbs_check_cpu(&od_dbs_data, cpu); + dbs_check_cpu(dbs_data, cpu); if (core_dbs_info->freq_lo) { /* Setup timer for SUB_SAMPLE */ core_dbs_info->sample_type = OD_SUB_SAMPLE; delay = core_dbs_info->freq_hi_jiffies; - } else { - delay = delay_for_sampling_rate(od_tuners.sampling_rate - * core_dbs_info->rate_mult); } } - schedule_delayed_work_on(smp_processor_id(), dw, delay); +max_delay: + if (!delay) + delay = delay_for_sampling_rate(od_tuners->sampling_rate + * core_dbs_info->rate_mult); + + gov_queue_work(dbs_data, dbs_info->cdbs.cur_policy, delay, modify_all); mutex_unlock(&core_dbs_info->cdbs.timer_mutex); } /************************** sysfs interface ************************/ - -static ssize_t show_sampling_rate_min(struct kobject *kobj, - struct attribute *attr, char *buf) -{ - return sprintf(buf, "%u\n", od_dbs_data.min_sampling_rate); -} +static struct common_dbs_data od_dbs_cdata; /** * update_sampling_rate - update sampling rate effective immediately if needed. @@ -276,12 +276,14 @@ static ssize_t show_sampling_rate_min(struct kobject *kobj, * reducing the sampling rate, we need to make the new value effective * immediately. */ -static void update_sampling_rate(unsigned int new_rate) +static void update_sampling_rate(struct dbs_data *dbs_data, + unsigned int new_rate) { + struct od_dbs_tuners *od_tuners = dbs_data->tuners; int cpu; - od_tuners.sampling_rate = new_rate = max(new_rate, - od_dbs_data.min_sampling_rate); + od_tuners->sampling_rate = new_rate = max(new_rate, + dbs_data->min_sampling_rate); for_each_online_cpu(cpu) { struct cpufreq_policy *policy; @@ -314,42 +316,54 @@ static void update_sampling_rate(unsigned int new_rate) cancel_delayed_work_sync(&dbs_info->cdbs.work); mutex_lock(&dbs_info->cdbs.timer_mutex); - schedule_delayed_work_on(cpu, &dbs_info->cdbs.work, - usecs_to_jiffies(new_rate)); + gov_queue_work(dbs_data, dbs_info->cdbs.cur_policy, + usecs_to_jiffies(new_rate), true); } mutex_unlock(&dbs_info->cdbs.timer_mutex); } } -static ssize_t store_sampling_rate(struct kobject *a, struct attribute *b, - const char *buf, size_t count) +static ssize_t store_sampling_rate(struct dbs_data *dbs_data, const char *buf, + size_t count) { unsigned int input; int ret; ret = sscanf(buf, "%u", &input); if (ret != 1) return -EINVAL; - update_sampling_rate(input); + + update_sampling_rate(dbs_data, input); return count; } -static ssize_t store_io_is_busy(struct kobject *a, struct attribute *b, - const char *buf, size_t count) +static ssize_t store_io_is_busy(struct dbs_data *dbs_data, const char *buf, + size_t count) { + struct od_dbs_tuners *od_tuners = dbs_data->tuners; unsigned int input; int ret; + unsigned int j; ret = sscanf(buf, "%u", &input); if (ret != 1) return -EINVAL; - od_tuners.io_is_busy = !!input; + od_tuners->io_is_busy = !!input; + + /* we need to re-evaluate prev_cpu_idle */ + for_each_online_cpu(j) { + struct od_cpu_dbs_info_s *dbs_info = &per_cpu(od_cpu_dbs_info, + j); + dbs_info->cdbs.prev_cpu_idle = get_cpu_idle_time(j, + &dbs_info->cdbs.prev_cpu_wall, od_tuners->io_is_busy); + } return count; } -static ssize_t store_up_threshold(struct kobject *a, struct attribute *b, - const char *buf, size_t count) +static ssize_t store_up_threshold(struct dbs_data *dbs_data, const char *buf, + size_t count) { + struct od_dbs_tuners *od_tuners = dbs_data->tuners; unsigned int input; int ret; ret = sscanf(buf, "%u", &input); @@ -359,23 +373,24 @@ static ssize_t store_up_threshold(struct kobject *a, struct attribute *b, return -EINVAL; } /* Calculate the new adj_up_threshold */ - od_tuners.adj_up_threshold += input; - od_tuners.adj_up_threshold -= od_tuners.up_threshold; + od_tuners->adj_up_threshold += input; + od_tuners->adj_up_threshold -= od_tuners->up_threshold; - od_tuners.up_threshold = input; + od_tuners->up_threshold = input; return count; } -static ssize_t store_sampling_down_factor(struct kobject *a, - struct attribute *b, const char *buf, size_t count) +static ssize_t store_sampling_down_factor(struct dbs_data *dbs_data, + const char *buf, size_t count) { + struct od_dbs_tuners *od_tuners = dbs_data->tuners; unsigned int input, j; int ret; ret = sscanf(buf, "%u", &input); if (ret != 1 || input > MAX_SAMPLING_DOWN_FACTOR || input < 1) return -EINVAL; - od_tuners.sampling_down_factor = input; + od_tuners->sampling_down_factor = input; /* Reset down sampling multiplier in case it was active */ for_each_online_cpu(j) { @@ -386,9 +401,10 @@ static ssize_t store_sampling_down_factor(struct kobject *a, return count; } -static ssize_t store_ignore_nice_load(struct kobject *a, struct attribute *b, - const char *buf, size_t count) +static ssize_t store_ignore_nice(struct dbs_data *dbs_data, const char *buf, + size_t count) { + struct od_dbs_tuners *od_tuners = dbs_data->tuners; unsigned int input; int ret; @@ -401,18 +417,18 @@ static ssize_t store_ignore_nice_load(struct kobject *a, struct attribute *b, if (input > 1) input = 1; - if (input == od_tuners.ignore_nice) { /* nothing to do */ + if (input == od_tuners->ignore_nice) { /* nothing to do */ return count; } - od_tuners.ignore_nice = input; + od_tuners->ignore_nice = input; /* we need to re-evaluate prev_cpu_idle */ for_each_online_cpu(j) { struct od_cpu_dbs_info_s *dbs_info; dbs_info = &per_cpu(od_cpu_dbs_info, j); dbs_info->cdbs.prev_cpu_idle = get_cpu_idle_time(j, - &dbs_info->cdbs.prev_cpu_wall); - if (od_tuners.ignore_nice) + &dbs_info->cdbs.prev_cpu_wall, od_tuners->io_is_busy); + if (od_tuners->ignore_nice) dbs_info->cdbs.prev_cpu_nice = kcpustat_cpu(j).cpustat[CPUTIME_NICE]; @@ -420,9 +436,10 @@ static ssize_t store_ignore_nice_load(struct kobject *a, struct attribute *b, return count; } -static ssize_t store_powersave_bias(struct kobject *a, struct attribute *b, - const char *buf, size_t count) +static ssize_t store_powersave_bias(struct dbs_data *dbs_data, const char *buf, + size_t count) { + struct od_dbs_tuners *od_tuners = dbs_data->tuners; unsigned int input; int ret; ret = sscanf(buf, "%u", &input); @@ -433,68 +450,179 @@ static ssize_t store_powersave_bias(struct kobject *a, struct attribute *b, if (input > 1000) input = 1000; - od_tuners.powersave_bias = input; + od_tuners->powersave_bias = input; ondemand_powersave_bias_init(); return count; } -show_one(od, sampling_rate, sampling_rate); -show_one(od, io_is_busy, io_is_busy); -show_one(od, up_threshold, up_threshold); -show_one(od, sampling_down_factor, sampling_down_factor); -show_one(od, ignore_nice_load, ignore_nice); -show_one(od, powersave_bias, powersave_bias); - -define_one_global_rw(sampling_rate); -define_one_global_rw(io_is_busy); -define_one_global_rw(up_threshold); -define_one_global_rw(sampling_down_factor); -define_one_global_rw(ignore_nice_load); -define_one_global_rw(powersave_bias); -define_one_global_ro(sampling_rate_min); - -static struct attribute *dbs_attributes[] = { - &sampling_rate_min.attr, - &sampling_rate.attr, - &up_threshold.attr, - &sampling_down_factor.attr, - &ignore_nice_load.attr, - &powersave_bias.attr, - &io_is_busy.attr, +show_store_one(od, sampling_rate); +show_store_one(od, io_is_busy); +show_store_one(od, up_threshold); +show_store_one(od, sampling_down_factor); +show_store_one(od, ignore_nice); +show_store_one(od, powersave_bias); +declare_show_sampling_rate_min(od); + +gov_sys_pol_attr_rw(sampling_rate); +gov_sys_pol_attr_rw(io_is_busy); +gov_sys_pol_attr_rw(up_threshold); +gov_sys_pol_attr_rw(sampling_down_factor); +gov_sys_pol_attr_rw(ignore_nice); +gov_sys_pol_attr_rw(powersave_bias); +gov_sys_pol_attr_ro(sampling_rate_min); + +static struct attribute *dbs_attributes_gov_sys[] = { + &sampling_rate_min_gov_sys.attr, + &sampling_rate_gov_sys.attr, + &up_threshold_gov_sys.attr, + &sampling_down_factor_gov_sys.attr, + &ignore_nice_gov_sys.attr, + &powersave_bias_gov_sys.attr, + &io_is_busy_gov_sys.attr, + NULL +}; + +static struct attribute_group od_attr_group_gov_sys = { + .attrs = dbs_attributes_gov_sys, + .name = "ondemand", +}; + +static struct attribute *dbs_attributes_gov_pol[] = { + &sampling_rate_min_gov_pol.attr, + &sampling_rate_gov_pol.attr, + &up_threshold_gov_pol.attr, + &sampling_down_factor_gov_pol.attr, + &ignore_nice_gov_pol.attr, + &powersave_bias_gov_pol.attr, + &io_is_busy_gov_pol.attr, NULL }; -static struct attribute_group od_attr_group = { - .attrs = dbs_attributes, +static struct attribute_group od_attr_group_gov_pol = { + .attrs = dbs_attributes_gov_pol, .name = "ondemand", }; /************************** sysfs end ************************/ +static int od_init(struct dbs_data *dbs_data) +{ + struct od_dbs_tuners *tuners; + u64 idle_time; + int cpu; + + tuners = kzalloc(sizeof(struct od_dbs_tuners), GFP_KERNEL); + if (!tuners) { + pr_err("%s: kzalloc failed\n", __func__); + return -ENOMEM; + } + + cpu = get_cpu(); + idle_time = get_cpu_idle_time_us(cpu, NULL); + put_cpu(); + if (idle_time != -1ULL) { + /* Idle micro accounting is supported. Use finer thresholds */ + tuners->up_threshold = MICRO_FREQUENCY_UP_THRESHOLD; + tuners->adj_up_threshold = MICRO_FREQUENCY_UP_THRESHOLD - + MICRO_FREQUENCY_DOWN_DIFFERENTIAL; + /* + * In nohz/micro accounting case we set the minimum frequency + * not depending on HZ, but fixed (very low). The deferred + * timer might skip some samples if idle/sleeping as needed. + */ + dbs_data->min_sampling_rate = MICRO_FREQUENCY_MIN_SAMPLE_RATE; + } else { + tuners->up_threshold = DEF_FREQUENCY_UP_THRESHOLD; + tuners->adj_up_threshold = DEF_FREQUENCY_UP_THRESHOLD - + DEF_FREQUENCY_DOWN_DIFFERENTIAL; + + /* For correct statistics, we need 10 ticks for each measure */ + dbs_data->min_sampling_rate = MIN_SAMPLING_RATE_RATIO * + jiffies_to_usecs(10); + } + + tuners->sampling_down_factor = DEF_SAMPLING_DOWN_FACTOR; + tuners->ignore_nice = 0; + tuners->powersave_bias = 0; + tuners->io_is_busy = should_io_be_busy(); + + dbs_data->tuners = tuners; + pr_info("%s: tuners %p\n", __func__, tuners); + mutex_init(&dbs_data->mutex); + return 0; +} + +static void od_exit(struct dbs_data *dbs_data) +{ + kfree(dbs_data->tuners); +} + define_get_cpu_dbs_routines(od_cpu_dbs_info); static struct od_ops od_ops = { - .io_busy = should_io_be_busy, .powersave_bias_init_cpu = ondemand_powersave_bias_init_cpu, - .powersave_bias_target = powersave_bias_target, + .powersave_bias_target = generic_powersave_bias_target, .freq_increase = dbs_freq_increase, }; -static struct dbs_data od_dbs_data = { +static struct common_dbs_data od_dbs_cdata = { .governor = GOV_ONDEMAND, - .attr_group = &od_attr_group, - .tuners = &od_tuners, + .attr_group_gov_sys = &od_attr_group_gov_sys, + .attr_group_gov_pol = &od_attr_group_gov_pol, .get_cpu_cdbs = get_cpu_cdbs, .get_cpu_dbs_info_s = get_cpu_dbs_info_s, .gov_dbs_timer = od_dbs_timer, .gov_check_cpu = od_check_cpu, .gov_ops = &od_ops, + .init = od_init, + .exit = od_exit, }; +static void od_set_powersave_bias(unsigned int powersave_bias) +{ + struct cpufreq_policy *policy; + struct dbs_data *dbs_data; + struct od_dbs_tuners *od_tuners; + unsigned int cpu; + cpumask_t done; + + cpumask_clear(&done); + + get_online_cpus(); + for_each_online_cpu(cpu) { + if (cpumask_test_cpu(cpu, &done)) + continue; + + policy = per_cpu(od_cpu_dbs_info, cpu).cdbs.cur_policy; + dbs_data = policy->governor_data; + od_tuners = dbs_data->tuners; + od_tuners->powersave_bias = powersave_bias; + + cpumask_or(&done, &done, policy->cpus); + } + put_online_cpus(); +} + +void od_register_powersave_bias_handler(unsigned int (*f) + (struct cpufreq_policy *, unsigned int, unsigned int), + unsigned int powersave_bias) +{ + od_ops.powersave_bias_target = f; + od_set_powersave_bias(powersave_bias); +} +EXPORT_SYMBOL_GPL(od_register_powersave_bias_handler); + +void od_unregister_powersave_bias_handler(void) +{ + od_ops.powersave_bias_target = generic_powersave_bias_target; + od_set_powersave_bias(0); +} +EXPORT_SYMBOL_GPL(od_unregister_powersave_bias_handler); + static int od_cpufreq_governor_dbs(struct cpufreq_policy *policy, unsigned int event) { - return cpufreq_governor_dbs(&od_dbs_data, policy, event); + return cpufreq_governor_dbs(policy, &od_dbs_cdata, event); } #ifndef CONFIG_CPU_FREQ_DEFAULT_GOV_ONDEMAND @@ -509,29 +637,6 @@ struct cpufreq_governor cpufreq_gov_ondemand = { static int __init cpufreq_gov_dbs_init(void) { - u64 idle_time; - int cpu = get_cpu(); - - mutex_init(&od_dbs_data.mutex); - idle_time = get_cpu_idle_time_us(cpu, NULL); - put_cpu(); - if (idle_time != -1ULL) { - /* Idle micro accounting is supported. Use finer thresholds */ - od_tuners.up_threshold = MICRO_FREQUENCY_UP_THRESHOLD; - od_tuners.adj_up_threshold = MICRO_FREQUENCY_UP_THRESHOLD - - MICRO_FREQUENCY_DOWN_DIFFERENTIAL; - /* - * In nohz/micro accounting case we set the minimum frequency - * not depending on HZ, but fixed (very low). The deferred - * timer might skip some samples if idle/sleeping as needed. - */ - od_dbs_data.min_sampling_rate = MICRO_FREQUENCY_MIN_SAMPLE_RATE; - } else { - /* For correct statistics, we need 10 ticks for each measure */ - od_dbs_data.min_sampling_rate = MIN_SAMPLING_RATE_RATIO * - jiffies_to_usecs(10); - } - return cpufreq_register_governor(&cpufreq_gov_ondemand); } |