diff options
Diffstat (limited to 'crypto')
| -rw-r--r-- | crypto/pcrypt.c | 191 |
1 files changed, 137 insertions, 54 deletions
diff --git a/crypto/pcrypt.c b/crypto/pcrypt.c index 6036b6de907..c9662e25595 100644 --- a/crypto/pcrypt.c +++ b/crypto/pcrypt.c @@ -24,12 +24,38 @@ #include <linux/init.h> #include <linux/module.h> #include <linux/slab.h> +#include <linux/notifier.h> #include <crypto/pcrypt.h> -static struct padata_instance *pcrypt_enc_padata; -static struct padata_instance *pcrypt_dec_padata; -static struct workqueue_struct *encwq; -static struct workqueue_struct *decwq; +struct pcrypt_instance { + struct padata_instance *pinst; + struct workqueue_struct *wq; + + /* + * Cpumask for callback CPUs. It should be + * equal to serial cpumask of corresponding padata instance, + * so it is updated when padata notifies us about serial + * cpumask change. + * + * cb_cpumask is protected by RCU. This fact prevents us from + * using cpumask_var_t directly because the actual type of + * cpumsak_var_t depends on kernel configuration(particularly on + * CONFIG_CPUMASK_OFFSTACK macro). Depending on the configuration + * cpumask_var_t may be either a pointer to the struct cpumask + * or a variable allocated on the stack. Thus we can not safely use + * cpumask_var_t with RCU operations such as rcu_assign_pointer or + * rcu_dereference. So cpumask_var_t is wrapped with struct + * pcrypt_cpumask which makes possible to use it with RCU. + */ + struct pcrypt_cpumask { + cpumask_var_t mask; + } *cb_cpumask; + struct notifier_block nblock; +}; + +static struct pcrypt_instance pencrypt; +static struct pcrypt_instance pdecrypt; + struct pcrypt_instance_ctx { struct crypto_spawn spawn; @@ -42,25 +68,29 @@ struct pcrypt_aead_ctx { }; static int pcrypt_do_parallel(struct padata_priv *padata, unsigned int *cb_cpu, - struct padata_instance *pinst) + struct pcrypt_instance *pcrypt) { unsigned int cpu_index, cpu, i; + struct pcrypt_cpumask *cpumask; cpu = *cb_cpu; - if (cpumask_test_cpu(cpu, cpu_active_mask)) + rcu_read_lock_bh(); + cpumask = rcu_dereference(pcrypt->cb_cpumask); + if (cpumask_test_cpu(cpu, cpumask->mask)) goto out; - cpu_index = cpu % cpumask_weight(cpu_active_mask); + cpu_index = cpu % cpumask_weight(cpumask->mask); - cpu = cpumask_first(cpu_active_mask); + cpu = cpumask_first(cpumask->mask); for (i = 0; i < cpu_index; i++) - cpu = cpumask_next(cpu, cpu_active_mask); + cpu = cpumask_next(cpu, cpumask->mask); *cb_cpu = cpu; out: - return padata_do_parallel(pinst, padata, cpu); + rcu_read_unlock_bh(); + return padata_do_parallel(pcrypt->pinst, padata, cpu); } static int pcrypt_aead_setkey(struct crypto_aead *parent, @@ -142,7 +172,7 @@ static int pcrypt_aead_encrypt(struct aead_request *req) req->cryptlen, req->iv); aead_request_set_assoc(creq, req->assoc, req->assoclen); - err = pcrypt_do_parallel(padata, &ctx->cb_cpu, pcrypt_enc_padata); + err = pcrypt_do_parallel(padata, &ctx->cb_cpu, &pencrypt); if (!err) return -EINPROGRESS; @@ -184,7 +214,7 @@ static int pcrypt_aead_decrypt(struct aead_request *req) req->cryptlen, req->iv); aead_request_set_assoc(creq, req->assoc, req->assoclen); - err = pcrypt_do_parallel(padata, &ctx->cb_cpu, pcrypt_dec_padata); + err = pcrypt_do_parallel(padata, &ctx->cb_cpu, &pdecrypt); if (!err) return -EINPROGRESS; @@ -228,7 +258,7 @@ static int pcrypt_aead_givencrypt(struct aead_givcrypt_request *req) aead_givcrypt_set_assoc(creq, areq->assoc, areq->assoclen); aead_givcrypt_set_giv(creq, req->giv, req->seq); - err = pcrypt_do_parallel(padata, &ctx->cb_cpu, pcrypt_enc_padata); + err = pcrypt_do_parallel(padata, &ctx->cb_cpu, &pencrypt); if (!err) return -EINPROGRESS; @@ -370,6 +400,88 @@ static void pcrypt_free(struct crypto_instance *inst) kfree(inst); } +static int pcrypt_cpumask_change_notify(struct notifier_block *self, + unsigned long val, void *data) +{ + struct pcrypt_instance *pcrypt; + struct pcrypt_cpumask *new_mask, *old_mask; + + if (!(val & PADATA_CPU_SERIAL)) + return 0; + + pcrypt = container_of(self, struct pcrypt_instance, nblock); + new_mask = kmalloc(sizeof(*new_mask), GFP_KERNEL); + if (!new_mask) + return -ENOMEM; + if (!alloc_cpumask_var(&new_mask->mask, GFP_KERNEL)) { + kfree(new_mask); + return -ENOMEM; + } + + old_mask = pcrypt->cb_cpumask; + + padata_get_cpumask(pcrypt->pinst, PADATA_CPU_SERIAL, new_mask->mask); + rcu_assign_pointer(pcrypt->cb_cpumask, new_mask); + synchronize_rcu_bh(); + + free_cpumask_var(old_mask->mask); + kfree(old_mask); + return 0; +} + +static int __pcrypt_init_instance(struct pcrypt_instance *pcrypt, + const char *name) +{ + int ret = -ENOMEM; + struct pcrypt_cpumask *mask; + + pcrypt->wq = create_workqueue(name); + if (!pcrypt->wq) + goto err; + + pcrypt->pinst = padata_alloc(pcrypt->wq); + if (!pcrypt->pinst) + goto err_destroy_workqueue; + + mask = kmalloc(sizeof(*mask), GFP_KERNEL); + if (!mask) + goto err_free_padata; + if (!alloc_cpumask_var(&mask->mask, GFP_KERNEL)) { + kfree(mask); + goto err_free_padata; + } + + padata_get_cpumask(pcrypt->pinst, PADATA_CPU_SERIAL, mask->mask); + rcu_assign_pointer(pcrypt->cb_cpumask, mask); + + pcrypt->nblock.notifier_call = pcrypt_cpumask_change_notify; + ret = padata_register_cpumask_notifier(pcrypt->pinst, &pcrypt->nblock); + if (ret) + goto err_free_cpumask; + + return ret; +err_free_cpumask: + free_cpumask_var(mask->mask); + kfree(mask); +err_free_padata: + padata_free(pcrypt->pinst); +err_destroy_workqueue: + destroy_workqueue(pcrypt->wq); +err: + return ret; +} + +static void __pcrypt_deinit_instance(struct pcrypt_instance *pcrypt) +{ + free_cpumask_var(pcrypt->cb_cpumask->mask); + kfree(pcrypt->cb_cpumask); + + padata_stop(pcrypt->pinst); + padata_unregister_cpumask_notifier(pcrypt->pinst, &pcrypt->nblock); + destroy_workqueue(pcrypt->wq); + padata_free(pcrypt->pinst); +} + static struct crypto_template pcrypt_tmpl = { .name = "pcrypt", .alloc = pcrypt_alloc, @@ -379,60 +491,31 @@ static struct crypto_template pcrypt_tmpl = { static int __init pcrypt_init(void) { - int err = -ENOMEM; - encwq = create_workqueue("pencrypt"); - if (!encwq) - goto err; - - decwq = create_workqueue("pdecrypt"); - if (!decwq) - goto err_destroy_encwq; - - - pcrypt_enc_padata = padata_alloc(cpu_possible_mask, encwq); - if (!pcrypt_enc_padata) - goto err_destroy_decwq; - - pcrypt_dec_padata = padata_alloc(cpu_possible_mask, decwq); - if (!pcrypt_dec_padata) - goto err_free_enc_padata; + int err; - err = padata_start(pcrypt_enc_padata); + err = __pcrypt_init_instance(&pencrypt, "pencrypt"); if (err) - goto err_free_dec_padata; + goto err; - err = padata_start(pcrypt_dec_padata); + err = __pcrypt_init_instance(&pdecrypt, "pdecrypt"); if (err) - goto err_free_dec_padata; - - return crypto_register_template(&pcrypt_tmpl); - -err_free_dec_padata: - padata_free(pcrypt_dec_padata); + goto err_deinit_pencrypt; -err_free_enc_padata: - padata_free(pcrypt_enc_padata); + padata_start(pencrypt.pinst); + padata_start(pdecrypt.pinst); -err_destroy_decwq: - destroy_workqueue(decwq); - -err_destroy_encwq: - destroy_workqueue(encwq); + return crypto_register_template(&pcrypt_tmpl); +err_deinit_pencrypt: + __pcrypt_deinit_instance(&pencrypt); err: return err; } static void __exit pcrypt_exit(void) { - padata_stop(pcrypt_enc_padata); - padata_stop(pcrypt_dec_padata); - - destroy_workqueue(encwq); - destroy_workqueue(decwq); - - padata_free(pcrypt_enc_padata); - padata_free(pcrypt_dec_padata); + __pcrypt_deinit_instance(&pencrypt); + __pcrypt_deinit_instance(&pdecrypt); crypto_unregister_template(&pcrypt_tmpl); } |