diff options
| -rw-r--r-- | include/linux/init_task.h | 3 | ||||
| -rw-r--r-- | include/linux/sched.h | 1 | ||||
| -rw-r--r-- | kernel/sched.c | 10 | ||||
| -rw-r--r-- | kernel/sched_fair.c | 21 |
4 files changed, 25 insertions, 10 deletions
diff --git a/include/linux/init_task.h b/include/linux/init_task.h index 1f74e1d7415..37a6f5bc4a9 100644 --- a/include/linux/init_task.h +++ b/include/linux/init_task.h @@ -151,6 +151,9 @@ extern struct group_info init_groups; .cpus_allowed = CPU_MASK_ALL, \ .mm = NULL, \ .active_mm = &init_mm, \ + .se = { \ + .group_node = LIST_HEAD_INIT(tsk.se.group_node), \ + }, \ .rt = { \ .run_list = LIST_HEAD_INIT(tsk.rt.run_list), \ .time_slice = HZ, \ diff --git a/include/linux/sched.h b/include/linux/sched.h index 887f5db8942..be6914014c7 100644 --- a/include/linux/sched.h +++ b/include/linux/sched.h @@ -946,6 +946,7 @@ struct load_weight { struct sched_entity { struct load_weight load; /* for load-balancing */ struct rb_node run_node; + struct list_head group_node; unsigned int on_rq; u64 exec_start; diff --git a/kernel/sched.c b/kernel/sched.c index ae1a3e936d2..3202462109f 100644 --- a/kernel/sched.c +++ b/kernel/sched.c @@ -384,8 +384,12 @@ struct cfs_rq { struct rb_root tasks_timeline; struct rb_node *rb_leftmost; - struct rb_node *rb_load_balance_curr; - /* 'curr' points to currently running entity on this cfs_rq. + + struct list_head tasks; + struct list_head *balance_iterator; + + /* + * 'curr' points to currently running entity on this cfs_rq. * It is set to NULL otherwise (i.e when none are currently running). */ struct sched_entity *curr, *next; @@ -2525,6 +2529,7 @@ static void __sched_fork(struct task_struct *p) INIT_LIST_HEAD(&p->rt.run_list); p->se.on_rq = 0; + INIT_LIST_HEAD(&p->se.group_node); #ifdef CONFIG_PREEMPT_NOTIFIERS INIT_HLIST_HEAD(&p->preempt_notifiers); @@ -7898,6 +7903,7 @@ int in_sched_functions(unsigned long addr) static void init_cfs_rq(struct cfs_rq *cfs_rq, struct rq *rq) { cfs_rq->tasks_timeline = RB_ROOT; + INIT_LIST_HEAD(&cfs_rq->tasks); #ifdef CONFIG_FAIR_GROUP_SCHED cfs_rq->rq = rq; #endif diff --git a/kernel/sched_fair.c b/kernel/sched_fair.c index 9e301a2bab6..ed8ce329899 100644 --- a/kernel/sched_fair.c +++ b/kernel/sched_fair.c @@ -533,6 +533,7 @@ account_entity_enqueue(struct cfs_rq *cfs_rq, struct sched_entity *se) add_cfs_task_weight(cfs_rq, se->load.weight); cfs_rq->nr_running++; se->on_rq = 1; + list_add(&se->group_node, &cfs_rq->tasks); } static void @@ -545,6 +546,7 @@ account_entity_dequeue(struct cfs_rq *cfs_rq, struct sched_entity *se) add_cfs_task_weight(cfs_rq, -se->load.weight); cfs_rq->nr_running--; se->on_rq = 0; + list_del_init(&se->group_node); } static void enqueue_sleeper(struct cfs_rq *cfs_rq, struct sched_entity *se) @@ -1289,21 +1291,24 @@ static void put_prev_task_fair(struct rq *rq, struct task_struct *prev) * the current task: */ static struct task_struct * -__load_balance_iterator(struct cfs_rq *cfs_rq, struct rb_node *curr) +__load_balance_iterator(struct cfs_rq *cfs_rq, struct list_head *next) { struct task_struct *p = NULL; struct sched_entity *se; - if (!curr) + if (next == &cfs_rq->tasks) return NULL; /* Skip over entities that are not tasks */ do { - se = rb_entry(curr, struct sched_entity, run_node); - curr = rb_next(curr); - } while (curr && !entity_is_task(se)); + se = list_entry(next, struct sched_entity, group_node); + next = next->next; + } while (next != &cfs_rq->tasks && !entity_is_task(se)); - cfs_rq->rb_load_balance_curr = curr; + if (next == &cfs_rq->tasks) + return NULL; + + cfs_rq->balance_iterator = next; if (entity_is_task(se)) p = task_of(se); @@ -1315,14 +1320,14 @@ static struct task_struct *load_balance_start_fair(void *arg) { struct cfs_rq *cfs_rq = arg; - return __load_balance_iterator(cfs_rq, first_fair(cfs_rq)); + return __load_balance_iterator(cfs_rq, cfs_rq->tasks.next); } static struct task_struct *load_balance_next_fair(void *arg) { struct cfs_rq *cfs_rq = arg; - return __load_balance_iterator(cfs_rq, cfs_rq->rb_load_balance_curr); + return __load_balance_iterator(cfs_rq, cfs_rq->balance_iterator); } static unsigned long |