diff options
| -rw-r--r-- | block/blk-cgroup.c | 150 | ||||
| -rw-r--r-- | block/blk-cgroup.h | 18 |
2 files changed, 102 insertions, 66 deletions
diff --git a/block/blk-cgroup.c b/block/blk-cgroup.c index 3b6a0e1265a..0eb39981e7c 100644 --- a/block/blk-cgroup.c +++ b/block/blk-cgroup.c @@ -184,12 +184,14 @@ static void blkio_check_and_dec_stat(uint64_t *stat, bool direction, bool sync) static void blkio_set_start_group_wait_time(struct blkio_group *blkg, struct blkio_group *curr_blkg) { - if (blkio_blkg_waiting(&blkg->stats)) + struct blkg_policy_data *pd = blkg->pd[blkg->plid]; + + if (blkio_blkg_waiting(&pd->stats)) return; if (blkg == curr_blkg) return; - blkg->stats.start_group_wait_time = sched_clock(); - blkio_mark_blkg_waiting(&blkg->stats); + pd->stats.start_group_wait_time = sched_clock(); + blkio_mark_blkg_waiting(&pd->stats); } /* This should be called with the blkg->stats_lock held. */ @@ -222,24 +224,26 @@ static void blkio_end_empty_time(struct blkio_group_stats *stats) void blkiocg_update_set_idle_time_stats(struct blkio_group *blkg) { + struct blkg_policy_data *pd = blkg->pd[blkg->plid]; unsigned long flags; spin_lock_irqsave(&blkg->stats_lock, flags); - BUG_ON(blkio_blkg_idling(&blkg->stats)); - blkg->stats.start_idle_time = sched_clock(); - blkio_mark_blkg_idling(&blkg->stats); + BUG_ON(blkio_blkg_idling(&pd->stats)); + pd->stats.start_idle_time = sched_clock(); + blkio_mark_blkg_idling(&pd->stats); spin_unlock_irqrestore(&blkg->stats_lock, flags); } EXPORT_SYMBOL_GPL(blkiocg_update_set_idle_time_stats); void blkiocg_update_idle_time_stats(struct blkio_group *blkg) { + struct blkg_policy_data *pd = blkg->pd[blkg->plid]; unsigned long flags; unsigned long long now; struct blkio_group_stats *stats; spin_lock_irqsave(&blkg->stats_lock, flags); - stats = &blkg->stats; + stats = &pd->stats; if (blkio_blkg_idling(stats)) { now = sched_clock(); if (time_after64(now, stats->start_idle_time)) @@ -252,11 +256,12 @@ EXPORT_SYMBOL_GPL(blkiocg_update_idle_time_stats); void blkiocg_update_avg_queue_size_stats(struct blkio_group *blkg) { + struct blkg_policy_data *pd = blkg->pd[blkg->plid]; unsigned long flags; struct blkio_group_stats *stats; spin_lock_irqsave(&blkg->stats_lock, flags); - stats = &blkg->stats; + stats = &pd->stats; stats->avg_queue_size_sum += stats->stat_arr[BLKIO_STAT_QUEUED][BLKIO_STAT_READ] + stats->stat_arr[BLKIO_STAT_QUEUED][BLKIO_STAT_WRITE]; @@ -268,11 +273,12 @@ EXPORT_SYMBOL_GPL(blkiocg_update_avg_queue_size_stats); void blkiocg_set_start_empty_time(struct blkio_group *blkg) { + struct blkg_policy_data *pd = blkg->pd[blkg->plid]; unsigned long flags; struct blkio_group_stats *stats; spin_lock_irqsave(&blkg->stats_lock, flags); - stats = &blkg->stats; + stats = &pd->stats; if (stats->stat_arr[BLKIO_STAT_QUEUED][BLKIO_STAT_READ] || stats->stat_arr[BLKIO_STAT_QUEUED][BLKIO_STAT_WRITE]) { @@ -299,7 +305,9 @@ EXPORT_SYMBOL_GPL(blkiocg_set_start_empty_time); void blkiocg_update_dequeue_stats(struct blkio_group *blkg, unsigned long dequeue) { - blkg->stats.dequeue += dequeue; + struct blkg_policy_data *pd = blkg->pd[blkg->plid]; + + pd->stats.dequeue += dequeue; } EXPORT_SYMBOL_GPL(blkiocg_update_dequeue_stats); #else @@ -312,12 +320,13 @@ void blkiocg_update_io_add_stats(struct blkio_group *blkg, struct blkio_group *curr_blkg, bool direction, bool sync) { + struct blkg_policy_data *pd = blkg->pd[blkg->plid]; unsigned long flags; spin_lock_irqsave(&blkg->stats_lock, flags); - blkio_add_stat(blkg->stats.stat_arr[BLKIO_STAT_QUEUED], 1, direction, + blkio_add_stat(pd->stats.stat_arr[BLKIO_STAT_QUEUED], 1, direction, sync); - blkio_end_empty_time(&blkg->stats); + blkio_end_empty_time(&pd->stats); blkio_set_start_group_wait_time(blkg, curr_blkg); spin_unlock_irqrestore(&blkg->stats_lock, flags); } @@ -326,10 +335,11 @@ EXPORT_SYMBOL_GPL(blkiocg_update_io_add_stats); void blkiocg_update_io_remove_stats(struct blkio_group *blkg, bool direction, bool sync) { + struct blkg_policy_data *pd = blkg->pd[blkg->plid]; unsigned long flags; spin_lock_irqsave(&blkg->stats_lock, flags); - blkio_check_and_dec_stat(blkg->stats.stat_arr[BLKIO_STAT_QUEUED], + blkio_check_and_dec_stat(pd->stats.stat_arr[BLKIO_STAT_QUEUED], direction, sync); spin_unlock_irqrestore(&blkg->stats_lock, flags); } @@ -338,12 +348,13 @@ EXPORT_SYMBOL_GPL(blkiocg_update_io_remove_stats); void blkiocg_update_timeslice_used(struct blkio_group *blkg, unsigned long time, unsigned long unaccounted_time) { + struct blkg_policy_data *pd = blkg->pd[blkg->plid]; unsigned long flags; spin_lock_irqsave(&blkg->stats_lock, flags); - blkg->stats.time += time; + pd->stats.time += time; #ifdef CONFIG_DEBUG_BLK_CGROUP - blkg->stats.unaccounted_time += unaccounted_time; + pd->stats.unaccounted_time += unaccounted_time; #endif spin_unlock_irqrestore(&blkg->stats_lock, flags); } @@ -356,6 +367,7 @@ EXPORT_SYMBOL_GPL(blkiocg_update_timeslice_used); void blkiocg_update_dispatch_stats(struct blkio_group *blkg, uint64_t bytes, bool direction, bool sync) { + struct blkg_policy_data *pd = blkg->pd[blkg->plid]; struct blkio_group_stats_cpu *stats_cpu; unsigned long flags; @@ -366,7 +378,7 @@ void blkiocg_update_dispatch_stats(struct blkio_group *blkg, */ local_irq_save(flags); - stats_cpu = this_cpu_ptr(blkg->stats_cpu); + stats_cpu = this_cpu_ptr(pd->stats_cpu); u64_stats_update_begin(&stats_cpu->syncp); stats_cpu->sectors += bytes >> 9; @@ -382,12 +394,13 @@ EXPORT_SYMBOL_GPL(blkiocg_update_dispatch_stats); void blkiocg_update_completion_stats(struct blkio_group *blkg, uint64_t start_time, uint64_t io_start_time, bool direction, bool sync) { + struct blkg_policy_data *pd = blkg->pd[blkg->plid]; struct blkio_group_stats *stats; unsigned long flags; unsigned long long now = sched_clock(); spin_lock_irqsave(&blkg->stats_lock, flags); - stats = &blkg->stats; + stats = &pd->stats; if (time_after64(now, io_start_time)) blkio_add_stat(stats->stat_arr[BLKIO_STAT_SERVICE_TIME], now - io_start_time, direction, sync); @@ -402,6 +415,7 @@ EXPORT_SYMBOL_GPL(blkiocg_update_completion_stats); void blkiocg_update_io_merged_stats(struct blkio_group *blkg, bool direction, bool sync) { + struct blkg_policy_data *pd = blkg->pd[blkg->plid]; struct blkio_group_stats_cpu *stats_cpu; unsigned long flags; @@ -412,7 +426,7 @@ void blkiocg_update_io_merged_stats(struct blkio_group *blkg, bool direction, */ local_irq_save(flags); - stats_cpu = this_cpu_ptr(blkg->stats_cpu); + stats_cpu = this_cpu_ptr(pd->stats_cpu); u64_stats_update_begin(&stats_cpu->syncp); blkio_add_stat(stats_cpu->stat_arr_cpu[BLKIO_STAT_CPU_MERGED], 1, @@ -430,11 +444,17 @@ EXPORT_SYMBOL_GPL(blkiocg_update_io_merged_stats); */ static void blkg_free(struct blkio_group *blkg) { - if (blkg) { - free_percpu(blkg->stats_cpu); - kfree(blkg->pd); - kfree(blkg); + struct blkg_policy_data *pd; + + if (!blkg) + return; + + pd = blkg->pd[blkg->plid]; + if (pd) { + free_percpu(pd->stats_cpu); + kfree(pd); } + kfree(blkg); } /** @@ -453,6 +473,7 @@ static struct blkio_group *blkg_alloc(struct blkio_cgroup *blkcg, struct blkio_policy_type *pol) { struct blkio_group *blkg; + struct blkg_policy_data *pd; /* alloc and init base part */ blkg = kzalloc_node(sizeof(*blkg), GFP_ATOMIC, q->node); @@ -466,23 +487,26 @@ static struct blkio_group *blkg_alloc(struct blkio_cgroup *blkcg, blkg->refcnt = 1; cgroup_path(blkcg->css.cgroup, blkg->path, sizeof(blkg->path)); - /* alloc per-policy data */ - blkg->pd = kzalloc_node(sizeof(*blkg->pd) + pol->pdata_size, GFP_ATOMIC, - q->node); - if (!blkg->pd) { + /* alloc per-policy data and attach it to blkg */ + pd = kzalloc_node(sizeof(*pd) + pol->pdata_size, GFP_ATOMIC, + q->node); + if (!pd) { blkg_free(blkg); return NULL; } + blkg->pd[pol->plid] = pd; + pd->blkg = blkg; + /* broken, read comment in the callsite */ - blkg->stats_cpu = alloc_percpu(struct blkio_group_stats_cpu); - if (!blkg->stats_cpu) { + + pd->stats_cpu = alloc_percpu(struct blkio_group_stats_cpu); + if (!pd->stats_cpu) { blkg_free(blkg); return NULL; } - /* attach pd to blkg and invoke per-policy init */ - blkg->pd->blkg = blkg; + /* invoke per-policy init */ pol->ops.blkio_init_group_fn(blkg); return blkg; } @@ -659,6 +683,7 @@ EXPORT_SYMBOL_GPL(__blkg_release); static void blkio_reset_stats_cpu(struct blkio_group *blkg) { + struct blkg_policy_data *pd = blkg->pd[blkg->plid]; struct blkio_group_stats_cpu *stats_cpu; int i, j, k; /* @@ -673,7 +698,7 @@ static void blkio_reset_stats_cpu(struct blkio_group *blkg) * unless this becomes a real issue. */ for_each_possible_cpu(i) { - stats_cpu = per_cpu_ptr(blkg->stats_cpu, i); + stats_cpu = per_cpu_ptr(pd->stats_cpu, i); stats_cpu->sectors = 0; for(j = 0; j < BLKIO_STAT_CPU_NR; j++) for (k = 0; k < BLKIO_STAT_TOTAL; k++) @@ -698,8 +723,10 @@ blkiocg_reset_stats(struct cgroup *cgroup, struct cftype *cftype, u64 val) blkcg = cgroup_to_blkio_cgroup(cgroup); spin_lock_irq(&blkcg->lock); hlist_for_each_entry(blkg, n, &blkcg->blkg_list, blkcg_node) { + struct blkg_policy_data *pd = blkg->pd[blkg->plid]; + spin_lock(&blkg->stats_lock); - stats = &blkg->stats; + stats = &pd->stats; #ifdef CONFIG_DEBUG_BLK_CGROUP idling = blkio_blkg_idling(stats); waiting = blkio_blkg_waiting(stats); @@ -779,13 +806,14 @@ static uint64_t blkio_fill_stat(char *str, int chars_left, uint64_t val, static uint64_t blkio_read_stat_cpu(struct blkio_group *blkg, enum stat_type_cpu type, enum stat_sub_type sub_type) { + struct blkg_policy_data *pd = blkg->pd[blkg->plid]; int cpu; struct blkio_group_stats_cpu *stats_cpu; u64 val = 0, tval; for_each_possible_cpu(cpu) { unsigned int start; - stats_cpu = per_cpu_ptr(blkg->stats_cpu, cpu); + stats_cpu = per_cpu_ptr(pd->stats_cpu, cpu); do { start = u64_stats_fetch_begin(&stats_cpu->syncp); @@ -837,20 +865,21 @@ static uint64_t blkio_get_stat(struct blkio_group *blkg, struct cgroup_map_cb *cb, const char *dname, enum stat_type type) { + struct blkg_policy_data *pd = blkg->pd[blkg->plid]; uint64_t disk_total; char key_str[MAX_KEY_LEN]; enum stat_sub_type sub_type; if (type == BLKIO_STAT_TIME) return blkio_fill_stat(key_str, MAX_KEY_LEN - 1, - blkg->stats.time, cb, dname); + pd->stats.time, cb, dname); #ifdef CONFIG_DEBUG_BLK_CGROUP if (type == BLKIO_STAT_UNACCOUNTED_TIME) return blkio_fill_stat(key_str, MAX_KEY_LEN - 1, - blkg->stats.unaccounted_time, cb, dname); + pd->stats.unaccounted_time, cb, dname); if (type == BLKIO_STAT_AVG_QUEUE_SIZE) { - uint64_t sum = blkg->stats.avg_queue_size_sum; - uint64_t samples = blkg->stats.avg_queue_size_samples; + uint64_t sum = pd->stats.avg_queue_size_sum; + uint64_t samples = pd->stats.avg_queue_size_samples; if (samples) do_div(sum, samples); else @@ -860,26 +889,26 @@ static uint64_t blkio_get_stat(struct blkio_group *blkg, } if (type == BLKIO_STAT_GROUP_WAIT_TIME) return blkio_fill_stat(key_str, MAX_KEY_LEN - 1, - blkg->stats.group_wait_time, cb, dname); + pd->stats.group_wait_time, cb, dname); if (type == BLKIO_STAT_IDLE_TIME) return blkio_fill_stat(key_str, MAX_KEY_LEN - 1, - blkg->stats.idle_time, cb, dname); + pd->stats.idle_time, cb, dname); if (type == BLKIO_STAT_EMPTY_TIME) return blkio_fill_stat(key_str, MAX_KEY_LEN - 1, - blkg->stats.empty_time, cb, dname); + pd->stats.empty_time, cb, dname); if (type == BLKIO_STAT_DEQUEUE) return blkio_fill_stat(key_str, MAX_KEY_LEN - 1, - blkg->stats.dequeue, cb, dname); + pd->stats.dequeue, cb, dname); #endif for (sub_type = BLKIO_STAT_READ; sub_type < BLKIO_STAT_TOTAL; sub_type++) { blkio_get_key_name(sub_type, dname, key_str, MAX_KEY_LEN, false); - cb->fill(cb, key_str, blkg->stats.stat_arr[type][sub_type]); + cb->fill(cb, key_str, pd->stats.stat_arr[type][sub_type]); } - disk_total = blkg->stats.stat_arr[type][BLKIO_STAT_READ] + - blkg->stats.stat_arr[type][BLKIO_STAT_WRITE]; + disk_total = pd->stats.stat_arr[type][BLKIO_STAT_READ] + + pd->stats.stat_arr[type][BLKIO_STAT_WRITE]; blkio_get_key_name(BLKIO_STAT_TOTAL, dname, key_str, MAX_KEY_LEN, false); cb->fill(cb, key_str, disk_total); @@ -891,6 +920,7 @@ static int blkio_policy_parse_and_set(char *buf, enum blkio_policy_id plid, { struct gendisk *disk = NULL; struct blkio_group *blkg = NULL; + struct blkg_policy_data *pd; char *s[4], *p, *major_s = NULL, *minor_s = NULL; unsigned long major, minor; int i = 0, ret = -EINVAL; @@ -950,35 +980,37 @@ static int blkio_policy_parse_and_set(char *buf, enum blkio_policy_id plid, goto out_unlock; } + pd = blkg->pd[plid]; + switch (plid) { case BLKIO_POLICY_PROP: if ((temp < BLKIO_WEIGHT_MIN && temp > 0) || temp > BLKIO_WEIGHT_MAX) goto out_unlock; - blkg->conf.weight = temp; + pd->conf.weight = temp; blkio_update_group_weight(blkg, temp ?: blkcg->weight); break; case BLKIO_POLICY_THROTL: switch(fileid) { case BLKIO_THROTL_read_bps_device: - blkg->conf.bps[READ] = temp; + pd->conf.bps[READ] = temp; blkio_update_group_bps(blkg, temp ?: -1, fileid); break; case BLKIO_THROTL_write_bps_device: - blkg->conf.bps[WRITE] = temp; + pd->conf.bps[WRITE] = temp; blkio_update_group_bps(blkg, temp ?: -1, fileid); break; case BLKIO_THROTL_read_iops_device: if (temp > THROTL_IOPS_MAX) goto out_unlock; - blkg->conf.iops[READ] = temp; + pd->conf.iops[READ] = temp; blkio_update_group_iops(blkg, temp ?: -1, fileid); break; case BLKIO_THROTL_write_iops_device: if (temp > THROTL_IOPS_MAX) goto out_unlock; - blkg->conf.iops[WRITE] = temp; + pd->conf.iops[WRITE] = temp; blkio_update_group_iops(blkg, temp ?: -1, fileid); break; } @@ -1034,6 +1066,7 @@ static const char *blkg_dev_name(struct blkio_group *blkg) static void blkio_print_group_conf(struct cftype *cft, struct blkio_group *blkg, struct seq_file *m) { + struct blkg_policy_data *pd = blkg->pd[blkg->plid]; const char *dname = blkg_dev_name(blkg); int fileid = BLKIOFILE_ATTR(cft->private); int rw = WRITE; @@ -1043,25 +1076,25 @@ static void blkio_print_group_conf(struct cftype *cft, struct blkio_group *blkg, switch (blkg->plid) { case BLKIO_POLICY_PROP: - if (blkg->conf.weight) + if (pd->conf.weight) seq_printf(m, "%s\t%u\n", - dname, blkg->conf.weight); + dname, pd->conf.weight); break; case BLKIO_POLICY_THROTL: switch (fileid) { case BLKIO_THROTL_read_bps_device: rw = READ; case BLKIO_THROTL_write_bps_device: - if (blkg->conf.bps[rw]) + if (pd->conf.bps[rw]) seq_printf(m, "%s\t%llu\n", - dname, blkg->conf.bps[rw]); + dname, pd->conf.bps[rw]); break; case BLKIO_THROTL_read_iops_device: rw = READ; case BLKIO_THROTL_write_iops_device: - if (blkg->conf.iops[rw]) + if (pd->conf.iops[rw]) seq_printf(m, "%s\t%u\n", - dname, blkg->conf.iops[rw]); + dname, pd->conf.iops[rw]); break; } break; @@ -1243,9 +1276,12 @@ static int blkio_weight_write(struct blkio_cgroup *blkcg, int plid, u64 val) spin_lock_irq(&blkcg->lock); blkcg->weight = (unsigned int)val; - hlist_for_each_entry(blkg, n, &blkcg->blkg_list, blkcg_node) - if (blkg->plid == plid && !blkg->conf.weight) + hlist_for_each_entry(blkg, n, &blkcg->blkg_list, blkcg_node) { + struct blkg_policy_data *pd = blkg->pd[blkg->plid]; + + if (blkg->plid == plid && !pd->conf.weight) blkio_update_group_weight(blkg, blkcg->weight); + } spin_unlock_irq(&blkcg->lock); spin_unlock(&blkio_list_lock); diff --git a/block/blk-cgroup.h b/block/blk-cgroup.h index 7da106843f0..5dffd436f30 100644 --- a/block/blk-cgroup.h +++ b/block/blk-cgroup.h @@ -164,6 +164,13 @@ struct blkg_policy_data { /* the blkg this per-policy data belongs to */ struct blkio_group *blkg; + /* Configuration */ + struct blkio_group_conf conf; + + struct blkio_group_stats stats; + /* Per cpu stats pointer */ + struct blkio_group_stats_cpu __percpu *stats_cpu; + /* pol->pdata_size bytes of private data used by policy impl */ char pdata[] __aligned(__alignof__(unsigned long long)); }; @@ -180,16 +187,9 @@ struct blkio_group { /* reference count */ int refcnt; - /* Configuration */ - struct blkio_group_conf conf; - /* Need to serialize the stats in the case of reset/update */ spinlock_t stats_lock; - struct blkio_group_stats stats; - /* Per cpu stats pointer */ - struct blkio_group_stats_cpu __percpu *stats_cpu; - - struct blkg_policy_data *pd; + struct blkg_policy_data *pd[BLKIO_NR_POLICIES]; struct rcu_head rcu_head; }; @@ -249,7 +249,7 @@ extern void blkg_destroy_all(struct request_queue *q); static inline void *blkg_to_pdata(struct blkio_group *blkg, struct blkio_policy_type *pol) { - return blkg ? blkg->pd->pdata : NULL; + return blkg ? blkg->pd[pol->plid]->pdata : NULL; } /** |