From 158e1645e07f3e9f7e4962d7a0997f5c3b98311b Mon Sep 17 00:00:00 2001 From: Al Viro Date: Wed, 27 Jun 2012 09:24:13 +0400 Subject: trim task_work: get rid of hlist layout based on Oleg's suggestion; single-linked list, task->task_works points to the last element, forward pointer from said last element points to head. I'd still prefer much more regular scheme with two pointers in task_work, but... Signed-off-by: Al Viro --- kernel/task_work.c | 64 ++++++++++++++++++++++++++++-------------------------- 1 file changed, 33 insertions(+), 31 deletions(-) (limited to 'kernel/task_work.c') diff --git a/kernel/task_work.c b/kernel/task_work.c index 82d1c794066..9b8948dbdc6 100644 --- a/kernel/task_work.c +++ b/kernel/task_work.c @@ -19,7 +19,12 @@ task_work_add(struct task_struct *task, struct task_work *twork, bool notify) */ raw_spin_lock_irqsave(&task->pi_lock, flags); if (likely(!(task->flags & PF_EXITING))) { - hlist_add_head(&twork->hlist, &task->task_works); + struct task_work *last = task->task_works; + struct task_work *first = last ? last->next : twork; + twork->next = first; + if (last) + last->next = twork; + task->task_works = twork; err = 0; } raw_spin_unlock_irqrestore(&task->pi_lock, flags); @@ -34,51 +39,48 @@ struct task_work * task_work_cancel(struct task_struct *task, task_work_func_t func) { unsigned long flags; - struct task_work *twork; - struct hlist_node *pos; + struct task_work *last, *res = NULL; raw_spin_lock_irqsave(&task->pi_lock, flags); - hlist_for_each_entry(twork, pos, &task->task_works, hlist) { - if (twork->func == func) { - hlist_del(&twork->hlist); - goto found; + last = task->task_works; + if (last) { + struct task_work *q = last, *p = q->next; + while (1) { + if (p->func == func) { + q->next = p->next; + if (p == last) + task->task_works = q == p ? NULL : q; + res = p; + break; + } + if (p == last) + break; + q = p; + p = q->next; } } - twork = NULL; - found: raw_spin_unlock_irqrestore(&task->pi_lock, flags); - - return twork; + return res; } void task_work_run(void) { struct task_struct *task = current; - struct hlist_head task_works; - struct hlist_node *pos; + struct task_work *p, *q; raw_spin_lock_irq(&task->pi_lock); - hlist_move_list(&task->task_works, &task_works); + p = task->task_works; + task->task_works = NULL; raw_spin_unlock_irq(&task->pi_lock); - if (unlikely(hlist_empty(&task_works))) + if (unlikely(!p)) return; - /* - * We use hlist to save the space in task_struct, but we want fifo. - * Find the last entry, the list should be short, then process them - * in reverse order. - */ - for (pos = task_works.first; pos->next; pos = pos->next) - ; - - for (;;) { - struct hlist_node **pprev = pos->pprev; - struct task_work *twork = container_of(pos, struct task_work, - hlist); - twork->func(twork); - if (pprev == &task_works.first) - break; - pos = container_of(pprev, struct hlist_node, next); + q = p->next; /* head */ + p->next = NULL; /* cut it */ + while (q) { + p = q->next; + q->func(q); + q = p; } } -- cgit v1.2.3-70-g09d2 From 67d1214551e800f9fe7dc7c47a346d2df0fafed5 Mon Sep 17 00:00:00 2001 From: Al Viro Date: Wed, 27 Jun 2012 11:07:19 +0400 Subject: merge task_work and rcu_head, get rid of separate allocation for keyring case task_work and rcu_head are identical now; merge them (calling the result struct callback_head, rcu_head #define'd to it), kill separate allocation in security/keys since we can just use cred->rcu now. Signed-off-by: Al Viro --- include/linux/sched.h | 2 +- include/linux/task_work.h | 14 ++++---------- include/linux/types.h | 9 +++++---- kernel/irq/manage.c | 4 ++-- kernel/task_work.c | 14 +++++++------- security/keys/internal.h | 6 +----- security/keys/keyctl.c | 28 +++++++++------------------- security/keys/process_keys.c | 6 ++---- 8 files changed, 31 insertions(+), 52 deletions(-) (limited to 'kernel/task_work.c') diff --git a/include/linux/sched.h b/include/linux/sched.h index b9216ebc278..af3555cc760 100644 --- a/include/linux/sched.h +++ b/include/linux/sched.h @@ -1405,7 +1405,7 @@ struct task_struct { int (*notifier)(void *priv); void *notifier_data; sigset_t *notifier_mask; - void *task_works; + struct callback_head *task_works; struct audit_context *audit_context; #ifdef CONFIG_AUDITSYSCALL diff --git a/include/linux/task_work.h b/include/linux/task_work.h index 3b3e2c8d037..fb46b03b185 100644 --- a/include/linux/task_work.h +++ b/include/linux/task_work.h @@ -4,22 +4,16 @@ #include #include -struct task_work; -typedef void (*task_work_func_t)(struct task_work *); - -struct task_work { - struct task_work *next; - task_work_func_t func; -}; +typedef void (*task_work_func_t)(struct callback_head *); static inline void -init_task_work(struct task_work *twork, task_work_func_t func) +init_task_work(struct callback_head *twork, task_work_func_t func) { twork->func = func; } -int task_work_add(struct task_struct *task, struct task_work *twork, bool); -struct task_work *task_work_cancel(struct task_struct *, task_work_func_t); +int task_work_add(struct task_struct *task, struct callback_head *twork, bool); +struct callback_head *task_work_cancel(struct task_struct *, task_work_func_t); void task_work_run(void); static inline void exit_task_work(struct task_struct *task) diff --git a/include/linux/types.h b/include/linux/types.h index 9c1bd539ea7..bf0dd7524b2 100644 --- a/include/linux/types.h +++ b/include/linux/types.h @@ -246,14 +246,15 @@ struct ustat { }; /** - * struct rcu_head - callback structure for use with RCU + * struct callback_head - callback structure for use with RCU and task_work * @next: next update requests in a list * @func: actual update function to call after the grace period. */ -struct rcu_head { - struct rcu_head *next; - void (*func)(struct rcu_head *head); +struct callback_head { + struct callback_head *next; + void (*func)(struct callback_head *head); }; +#define rcu_head callback_head #endif /* __KERNEL__ */ #endif /* __ASSEMBLY__ */ diff --git a/kernel/irq/manage.c b/kernel/irq/manage.c index d1dd54734ce..814c9ef6bba 100644 --- a/kernel/irq/manage.c +++ b/kernel/irq/manage.c @@ -781,7 +781,7 @@ static void wake_threads_waitq(struct irq_desc *desc) wake_up(&desc->wait_for_threads); } -static void irq_thread_dtor(struct task_work *unused) +static void irq_thread_dtor(struct callback_head *unused) { struct task_struct *tsk = current; struct irq_desc *desc; @@ -813,7 +813,7 @@ static void irq_thread_dtor(struct task_work *unused) */ static int irq_thread(void *data) { - struct task_work on_exit_work; + struct callback_head on_exit_work; static const struct sched_param param = { .sched_priority = MAX_USER_RT_PRIO/2, }; diff --git a/kernel/task_work.c b/kernel/task_work.c index 9b8948dbdc6..76266fb665d 100644 --- a/kernel/task_work.c +++ b/kernel/task_work.c @@ -3,7 +3,7 @@ #include int -task_work_add(struct task_struct *task, struct task_work *twork, bool notify) +task_work_add(struct task_struct *task, struct callback_head *twork, bool notify) { unsigned long flags; int err = -ESRCH; @@ -19,8 +19,8 @@ task_work_add(struct task_struct *task, struct task_work *twork, bool notify) */ raw_spin_lock_irqsave(&task->pi_lock, flags); if (likely(!(task->flags & PF_EXITING))) { - struct task_work *last = task->task_works; - struct task_work *first = last ? last->next : twork; + struct callback_head *last = task->task_works; + struct callback_head *first = last ? last->next : twork; twork->next = first; if (last) last->next = twork; @@ -35,16 +35,16 @@ task_work_add(struct task_struct *task, struct task_work *twork, bool notify) return err; } -struct task_work * +struct callback_head * task_work_cancel(struct task_struct *task, task_work_func_t func) { unsigned long flags; - struct task_work *last, *res = NULL; + struct callback_head *last, *res = NULL; raw_spin_lock_irqsave(&task->pi_lock, flags); last = task->task_works; if (last) { - struct task_work *q = last, *p = q->next; + struct callback_head *q = last, *p = q->next; while (1) { if (p->func == func) { q->next = p->next; @@ -66,7 +66,7 @@ task_work_cancel(struct task_struct *task, task_work_func_t func) void task_work_run(void) { struct task_struct *task = current; - struct task_work *p, *q; + struct callback_head *p, *q; raw_spin_lock_irq(&task->pi_lock); p = task->task_works; diff --git a/security/keys/internal.h b/security/keys/internal.h index b510a316874..c246ba5d43a 100644 --- a/security/keys/internal.h +++ b/security/keys/internal.h @@ -148,12 +148,8 @@ extern key_ref_t lookup_user_key(key_serial_t id, unsigned long flags, #define KEY_LOOKUP_PARTIAL 0x02 #define KEY_LOOKUP_FOR_UNLINK 0x04 -struct kludge { /* this will die off very soon */ - struct task_work twork; - struct cred *cred; -}; extern long join_session_keyring(const char *name); -extern void key_change_session_keyring(struct task_work *twork); +extern void key_change_session_keyring(struct callback_head *twork); extern struct work_struct key_gc_work; extern unsigned key_gc_delay; diff --git a/security/keys/keyctl.c b/security/keys/keyctl.c index 26723caaad0..0291b3f9397 100644 --- a/security/keys/keyctl.c +++ b/security/keys/keyctl.c @@ -1456,8 +1456,7 @@ long keyctl_session_to_parent(void) { struct task_struct *me, *parent; const struct cred *mycred, *pcred; - struct kludge *newwork; - struct task_work *oldwork; + struct callback_head *newwork, *oldwork; key_ref_t keyring_r; struct cred *cred; int ret; @@ -1467,20 +1466,17 @@ long keyctl_session_to_parent(void) return PTR_ERR(keyring_r); ret = -ENOMEM; - newwork = kmalloc(sizeof(struct kludge), GFP_KERNEL); - if (!newwork) - goto error_keyring; /* our parent is going to need a new cred struct, a new tgcred struct * and new security data, so we allocate them here to prevent ENOMEM in * our parent */ cred = cred_alloc_blank(); if (!cred) - goto error_newwork; + goto error_keyring; + newwork = &cred->rcu; cred->tgcred->session_keyring = key_ref_to_ptr(keyring_r); - init_task_work(&newwork->twork, key_change_session_keyring); - newwork->cred = cred; + init_task_work(newwork, key_change_session_keyring); me = current; rcu_read_lock(); @@ -1529,24 +1525,18 @@ long keyctl_session_to_parent(void) /* the replacement session keyring is applied just prior to userspace * restarting */ - ret = task_work_add(parent, &newwork->twork, true); + ret = task_work_add(parent, newwork, true); if (!ret) newwork = NULL; unlock: write_unlock_irq(&tasklist_lock); rcu_read_unlock(); - if (oldwork) { - put_cred(container_of(oldwork, struct kludge, twork)->cred); - kfree(oldwork); - } - if (newwork) { - put_cred(newwork->cred); - kfree(newwork); - } + if (oldwork) + put_cred(container_of(oldwork, struct cred, rcu)); + if (newwork) + put_cred(cred); return ret; -error_newwork: - kfree(newwork); error_keyring: key_ref_put(keyring_r); return ret; diff --git a/security/keys/process_keys.c b/security/keys/process_keys.c index c9b07c97d7f..54339cfd673 100644 --- a/security/keys/process_keys.c +++ b/security/keys/process_keys.c @@ -834,13 +834,11 @@ error: * Replace a process's session keyring on behalf of one of its children when * the target process is about to resume userspace execution. */ -void key_change_session_keyring(struct task_work *twork) +void key_change_session_keyring(struct callback_head *twork) { const struct cred *old = current_cred(); - struct kludge *p = container_of(twork, struct kludge, twork); - struct cred *new = p->cred; + struct cred *new = container_of(twork, struct cred, rcu); - kfree(p); if (unlikely(current->flags & PF_EXITING)) { put_cred(new); return; -- cgit v1.2.3-70-g09d2 From ed3e694d78cc75fa79bf29698631b146fd27aa35 Mon Sep 17 00:00:00 2001 From: Al Viro Date: Wed, 27 Jun 2012 11:31:24 +0400 Subject: move exit_task_work() past exit_files() et.al. ... and get rid of PF_EXITING check in task_work_add(). Signed-off-by: Al Viro --- kernel/exit.c | 6 ++---- kernel/task_work.c | 30 +++++++++++------------------- 2 files changed, 13 insertions(+), 23 deletions(-) (limited to 'kernel/task_work.c') diff --git a/kernel/exit.c b/kernel/exit.c index 2f59cc33451..d17f6c4ddfa 100644 --- a/kernel/exit.c +++ b/kernel/exit.c @@ -953,14 +953,11 @@ void do_exit(long code) exit_signals(tsk); /* sets PF_EXITING */ /* * tsk->flags are checked in the futex code to protect against - * an exiting task cleaning up the robust pi futexes, and in - * task_work_add() to avoid the race with exit_task_work(). + * an exiting task cleaning up the robust pi futexes. */ smp_mb(); raw_spin_unlock_wait(&tsk->pi_lock); - exit_task_work(tsk); - if (unlikely(in_atomic())) printk(KERN_INFO "note: %s[%d] exited with preempt_count %d\n", current->comm, task_pid_nr(current), @@ -995,6 +992,7 @@ void do_exit(long code) exit_shm(tsk); exit_files(tsk); exit_fs(tsk); + exit_task_work(tsk); check_stack_usage(); exit_thread(); diff --git a/kernel/task_work.c b/kernel/task_work.c index 76266fb665d..fb396089f66 100644 --- a/kernel/task_work.c +++ b/kernel/task_work.c @@ -5,34 +5,26 @@ int task_work_add(struct task_struct *task, struct callback_head *twork, bool notify) { + struct callback_head *last, *first; unsigned long flags; - int err = -ESRCH; -#ifndef TIF_NOTIFY_RESUME - if (notify) - return -ENOTSUPP; -#endif /* - * We must not insert the new work if the task has already passed - * exit_task_work(). We rely on do_exit()->raw_spin_unlock_wait() - * and check PF_EXITING under pi_lock. + * Not inserting the new work if the task has already passed + * exit_task_work() is the responisbility of callers. */ raw_spin_lock_irqsave(&task->pi_lock, flags); - if (likely(!(task->flags & PF_EXITING))) { - struct callback_head *last = task->task_works; - struct callback_head *first = last ? last->next : twork; - twork->next = first; - if (last) - last->next = twork; - task->task_works = twork; - err = 0; - } + last = task->task_works; + first = last ? last->next : twork; + twork->next = first; + if (last) + last->next = twork; + task->task_works = twork; raw_spin_unlock_irqrestore(&task->pi_lock, flags); /* test_and_set_bit() implies mb(), see tracehook_notify_resume(). */ - if (likely(!err) && notify) + if (notify) set_notify_resume(task); - return err; + return 0; } struct callback_head * -- cgit v1.2.3-70-g09d2 From a2d4c71d1559426155e5da8db3265bfa0d8d398d Mon Sep 17 00:00:00 2001 From: Al Viro Date: Wed, 27 Jun 2012 11:33:29 +0400 Subject: deal with task_work callbacks adding more work It doesn't matter on normal return to userland path (we'll recheck the NOTIFY_RESUME flag anyway), but in case of exit_task_work() we'll need that as soon as we get callbacks capable of triggering more task_work_add(). Signed-off-by: Al Viro --- kernel/task_work.c | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) (limited to 'kernel/task_work.c') diff --git a/kernel/task_work.c b/kernel/task_work.c index fb396089f66..91d4e1742a0 100644 --- a/kernel/task_work.c +++ b/kernel/task_work.c @@ -60,19 +60,21 @@ void task_work_run(void) struct task_struct *task = current; struct callback_head *p, *q; - raw_spin_lock_irq(&task->pi_lock); - p = task->task_works; - task->task_works = NULL; - raw_spin_unlock_irq(&task->pi_lock); + while (1) { + raw_spin_lock_irq(&task->pi_lock); + p = task->task_works; + task->task_works = NULL; + raw_spin_unlock_irq(&task->pi_lock); - if (unlikely(!p)) - return; + if (unlikely(!p)) + return; - q = p->next; /* head */ - p->next = NULL; /* cut it */ - while (q) { - p = q->next; - q->func(q); - q = p; + q = p->next; /* head */ + p->next = NULL; /* cut it */ + while (q) { + p = q->next; + q->func(q); + q = p; + } } } -- cgit v1.2.3-70-g09d2