From 4aaf087846f9a1f1ec272393f5cd78f713e24f37 Mon Sep 17 00:00:00 2001 From: Lee Nipper Date: Thu, 17 Jul 2008 15:58:08 +0800 Subject: crypto: talitos - Remove calls to of_node_put Remove of_node_put calls since there is no corresponding of_node_get. This patch prevents an exception when talitos is loaded a 2nd time. This sequence: modprobe talitos; rmmod talitos; modprobe talitos causes this message: "WARNING: Bad of_node_put() on /soc8349@e0000000/crypto@30000". Signed-off-by: Lee Nipper Signed-off-by: Kim Phillips Signed-off-by: Herbert Xu --- drivers/crypto/talitos.c | 5 ----- 1 file changed, 5 deletions(-) (limited to 'drivers/crypto/talitos.c') diff --git a/drivers/crypto/talitos.c b/drivers/crypto/talitos.c index b11943dadef..01e6595014e 100644 --- a/drivers/crypto/talitos.c +++ b/drivers/crypto/talitos.c @@ -1466,9 +1466,6 @@ static int talitos_probe(struct of_device *ofdev, goto err_out; } - of_node_put(np); - np = NULL; - priv->head_lock = kmalloc(sizeof(spinlock_t) * priv->num_channels, GFP_KERNEL); priv->tail_lock = kmalloc(sizeof(spinlock_t) * priv->num_channels, @@ -1559,8 +1556,6 @@ static int talitos_probe(struct of_device *ofdev, err_out: talitos_remove(ofdev); - if (np) - of_node_put(np); return err; } -- cgit v1.2.3-70-g09d2 From 695ad589698571046d42a4450c2d801486905535 Mon Sep 17 00:00:00 2001 From: Lee Nipper Date: Thu, 17 Jul 2008 16:22:30 +0800 Subject: crypto: talitos - Correct dst != src case handling Seems that dst == src, but this fixes the logic in case it's not. Signed-off-by: Lee Nipper Signed-off-by: Kim Phillips Signed-off-by: Herbert Xu --- drivers/crypto/talitos.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'drivers/crypto/talitos.c') diff --git a/drivers/crypto/talitos.c b/drivers/crypto/talitos.c index 01e6595014e..f644fba35c8 100644 --- a/drivers/crypto/talitos.c +++ b/drivers/crypto/talitos.c @@ -1022,7 +1022,7 @@ static struct ipsec_esp_edesc *ipsec_esp_edesc_alloc(struct aead_request *areq, dst_nents = src_nents; } else { dst_nents = sg_count(areq->dst, areq->cryptlen + ctx->authsize); - dst_nents = (dst_nents == 1) ? 0 : src_nents; + dst_nents = (dst_nents == 1) ? 0 : dst_nents; } /* -- cgit v1.2.3-70-g09d2 From ec6644d6325b5a38525f1d5b20fd4bf7db05cf2a Mon Sep 17 00:00:00 2001 From: Kim Phillips Date: Thu, 17 Jul 2008 20:16:40 +0800 Subject: crypto: talitos - Preempt overflow interrupts add requests pending/submit count to prevent request queue full condition by preempting h/w overflow interrupts in software. We do this due to the delay in the delivery and handling of the channel overflow error interrupt. Signed-off-by: Kim Phillips Acked-by: Lee Nipper Signed-off-by: Herbert Xu --- drivers/crypto/talitos.c | 27 ++++++++++++++++++++++----- 1 file changed, 22 insertions(+), 5 deletions(-) (limited to 'drivers/crypto/talitos.c') diff --git a/drivers/crypto/talitos.c b/drivers/crypto/talitos.c index f644fba35c8..fdb0680f6ff 100644 --- a/drivers/crypto/talitos.c +++ b/drivers/crypto/talitos.c @@ -99,6 +99,9 @@ struct talitos_private { /* next channel to be assigned next incoming descriptor */ atomic_t last_chan; + /* per-channel number of requests pending in channel h/w fifo */ + atomic_t *submit_count; + /* per-channel request fifo */ struct talitos_request **fifo; @@ -263,15 +266,15 @@ static int talitos_submit(struct device *dev, struct talitos_desc *desc, spin_lock_irqsave(&priv->head_lock[ch], flags); - head = priv->head[ch]; - request = &priv->fifo[ch][head]; - - if (request->desc) { - /* request queue is full */ + if (!atomic_inc_not_zero(&priv->submit_count[ch])) { + /* h/w fifo is full */ spin_unlock_irqrestore(&priv->head_lock[ch], flags); return -EAGAIN; } + head = priv->head[ch]; + request = &priv->fifo[ch][head]; + /* map descriptor and save caller data */ request->dma_desc = dma_map_single(dev, desc, sizeof(*desc), DMA_BIDIRECTIONAL); @@ -335,6 +338,9 @@ static void flush_channel(struct device *dev, int ch, int error, int reset_ch) priv->tail[ch] = (tail + 1) & (priv->fifo_len - 1); spin_unlock_irqrestore(&priv->tail_lock[ch], flags); + + atomic_dec(&priv->submit_count[ch]); + saved_req.callback(dev, saved_req.desc, saved_req.context, status); /* channel may resume processing in single desc error case */ @@ -1337,6 +1343,7 @@ static int __devexit talitos_remove(struct of_device *ofdev) if (hw_supports(dev, DESC_HDR_SEL0_RNG)) talitos_unregister_rng(dev); + kfree(priv->submit_count); kfree(priv->tail); kfree(priv->head); @@ -1501,6 +1508,16 @@ static int talitos_probe(struct of_device *ofdev, } } + priv->submit_count = kmalloc(sizeof(int) * priv->num_channels, + GFP_KERNEL); + if (!priv->submit_count) { + dev_err(dev, "failed to allocate fifo submit count space\n"); + err = -ENOMEM; + goto err_out; + } + for (i = 0; i < priv->num_channels; i++) + atomic_set(&priv->submit_count[i], -priv->chfifo_len); + priv->head = kzalloc(sizeof(int) * priv->num_channels, GFP_KERNEL); priv->tail = kzalloc(sizeof(int) * priv->num_channels, GFP_KERNEL); if (!priv->head || !priv->tail) { -- cgit v1.2.3-70-g09d2 From 586725f8604ef16ebbfdd66e73036e162ae00135 Mon Sep 17 00:00:00 2001 From: Kim Phillips Date: Thu, 17 Jul 2008 20:19:18 +0800 Subject: crypto: talitos - Fix GFP flag usage use GFP_ATOMIC when necessary; use atomic_t when allocating submit_count. Signed-off-by: Kim Phillips Acked-by: Lee Nipper Signed-off-by: Herbert Xu --- drivers/crypto/talitos.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) (limited to 'drivers/crypto/talitos.c') diff --git a/drivers/crypto/talitos.c b/drivers/crypto/talitos.c index fdb0680f6ff..79fdba2a20c 100644 --- a/drivers/crypto/talitos.c +++ b/drivers/crypto/talitos.c @@ -1015,6 +1015,8 @@ static struct ipsec_esp_edesc *ipsec_esp_edesc_alloc(struct aead_request *areq, struct talitos_ctx *ctx = crypto_aead_ctx(authenc); struct ipsec_esp_edesc *edesc; int src_nents, dst_nents, alloc_len, dma_len; + gfp_t flags = areq->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL : + GFP_ATOMIC; if (areq->cryptlen + ctx->authsize > TALITOS_MAX_DATA_LEN) { dev_err(ctx->dev, "cryptlen exceeds h/w max limit\n"); @@ -1046,7 +1048,7 @@ static struct ipsec_esp_edesc *ipsec_esp_edesc_alloc(struct aead_request *areq, alloc_len += icv_stashing ? ctx->authsize : 0; } - edesc = kmalloc(alloc_len, GFP_DMA); + edesc = kmalloc(alloc_len, GFP_DMA | flags); if (!edesc) { dev_err(ctx->dev, "could not allocate edescriptor\n"); return ERR_PTR(-ENOMEM); @@ -1508,7 +1510,7 @@ static int talitos_probe(struct of_device *ofdev, } } - priv->submit_count = kmalloc(sizeof(int) * priv->num_channels, + priv->submit_count = kmalloc(sizeof(atomic_t) * priv->num_channels, GFP_KERNEL); if (!priv->submit_count) { dev_err(dev, "failed to allocate fifo submit count space\n"); -- cgit v1.2.3-70-g09d2 From fa86a26795b850cdf4e557898457a63e241c1aa1 Mon Sep 17 00:00:00 2001 From: Kim Phillips Date: Thu, 17 Jul 2008 20:20:06 +0800 Subject: crypto: talitos - Stop leaking memory in error path free edescriptor when returning error (such as -EAGAIN). Signed-off-by: Kim Phillips Acked-by: Lee Nipper Signed-off-by: Herbert Xu --- drivers/crypto/talitos.c | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) (limited to 'drivers/crypto/talitos.c') diff --git a/drivers/crypto/talitos.c b/drivers/crypto/talitos.c index 79fdba2a20c..a81265bbb89 100644 --- a/drivers/crypto/talitos.c +++ b/drivers/crypto/talitos.c @@ -880,7 +880,7 @@ static int ipsec_esp(struct ipsec_esp_edesc *edesc, struct aead_request *areq, unsigned int cryptlen = areq->cryptlen; unsigned int authsize = ctx->authsize; unsigned int ivsize; - int sg_count; + int sg_count, ret; /* hmac key */ map_single_talitos_ptr(dev, &desc->ptr[0], ctx->authkeylen, &ctx->key, @@ -984,7 +984,12 @@ static int ipsec_esp(struct ipsec_esp_edesc *edesc, struct aead_request *areq, map_single_talitos_ptr(dev, &desc->ptr[6], ivsize, ctx->iv, 0, DMA_FROM_DEVICE); - return talitos_submit(dev, desc, callback, areq); + ret = talitos_submit(dev, desc, callback, areq); + if (ret != -EINPROGRESS) { + ipsec_esp_unmap(dev, edesc, areq); + kfree(edesc); + } + return ret; } -- cgit v1.2.3-70-g09d2 From c0e741d47859fcabb84a37589a4f49801ca8590a Mon Sep 17 00:00:00 2001 From: Kim Phillips Date: Thu, 17 Jul 2008 20:20:59 +0800 Subject: crypto: talitos - sparse fix Signed-off-by: Kim Phillips Signed-off-by: Herbert Xu --- drivers/crypto/talitos.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'drivers/crypto/talitos.c') diff --git a/drivers/crypto/talitos.c b/drivers/crypto/talitos.c index a81265bbb89..681c15f4208 100644 --- a/drivers/crypto/talitos.c +++ b/drivers/crypto/talitos.c @@ -848,7 +848,7 @@ static int sg_to_link_tbl(struct scatterlist *sg, int sg_count, /* adjust (decrease) last one (or two) entry's len to cryptlen */ link_tbl_ptr--; - while (link_tbl_ptr->len <= (-cryptlen)) { + while (be16_to_cpu(link_tbl_ptr->len) <= (-cryptlen)) { /* Empty this entry, and move to previous one */ cryptlen += be16_to_cpu(link_tbl_ptr->len); link_tbl_ptr->len = 0; -- cgit v1.2.3-70-g09d2 From f3c85bc1bc72b4cc8d58664a490a9d42bdb6565a Mon Sep 17 00:00:00 2001 From: Lee Nipper Date: Wed, 30 Jul 2008 16:26:57 +0800 Subject: crypto: talitos - Add handling for SEC 3.x treatment of link table Later SEC revision requires the link table (used for scatter/gather) to have an extra entry to account for the total length in descriptor [4], which contains cipher Input and ICV. This only applies to decrypt, not encrypt. Without this change, on 837x, a gather return/length error results when a decryption uses a link table to gather the fragments. This is observed by doing a ping with size of 1447 or larger with AES, or a ping with size 1455 or larger with 3des. So, add check for SEC compatible "fsl,3.0" for using extra link table entry. Signed-off-by: Lee Nipper Signed-off-by: Kim Phillips Signed-off-by: Herbert Xu --- drivers/crypto/talitos.c | 54 ++++++++++++++++++++++++++++++++++-------------- 1 file changed, 39 insertions(+), 15 deletions(-) (limited to 'drivers/crypto/talitos.c') diff --git a/drivers/crypto/talitos.c b/drivers/crypto/talitos.c index 681c15f4208..ee827a7f7c6 100644 --- a/drivers/crypto/talitos.c +++ b/drivers/crypto/talitos.c @@ -96,6 +96,9 @@ struct talitos_private { unsigned int exec_units; unsigned int desc_types; + /* SEC Compatibility info */ + unsigned long features; + /* next channel to be assigned next incoming descriptor */ atomic_t last_chan; @@ -133,6 +136,9 @@ struct talitos_private { struct hwrng rng; }; +/* .features flag */ +#define TALITOS_FTR_SRC_LINK_TBL_LEN_INCLUDES_EXTENT 0x00000001 + /* * map virtual single (contiguous) pointer to h/w descriptor pointer */ @@ -785,7 +791,7 @@ static void ipsec_esp_encrypt_done(struct device *dev, /* copy the generated ICV to dst */ if (edesc->dma_len) { icvdata = &edesc->link_tbl[edesc->src_nents + - edesc->dst_nents + 1]; + edesc->dst_nents + 2]; sg = sg_last(areq->dst, edesc->dst_nents); memcpy((char *)sg_virt(sg) + sg->length - ctx->authsize, icvdata, ctx->authsize); @@ -814,7 +820,7 @@ static void ipsec_esp_decrypt_done(struct device *dev, /* auth check */ if (edesc->dma_len) icvdata = &edesc->link_tbl[edesc->src_nents + - edesc->dst_nents + 1]; + edesc->dst_nents + 2]; else icvdata = &edesc->link_tbl[0]; @@ -921,10 +927,30 @@ static int ipsec_esp(struct ipsec_esp_edesc *edesc, struct aead_request *areq, sg_count = sg_to_link_tbl(areq->src, sg_count, cryptlen, &edesc->link_tbl[0]); if (sg_count > 1) { + struct talitos_ptr *link_tbl_ptr = + &edesc->link_tbl[sg_count-1]; + struct scatterlist *sg; + struct talitos_private *priv = dev_get_drvdata(dev); + desc->ptr[4].j_extent |= DESC_PTR_LNKTBL_JUMP; desc->ptr[4].ptr = cpu_to_be32(edesc->dma_link_tbl); dma_sync_single_for_device(ctx->dev, edesc->dma_link_tbl, edesc->dma_len, DMA_BIDIRECTIONAL); + /* If necessary for this SEC revision, + * add a link table entry for ICV. + */ + if ((priv->features & + TALITOS_FTR_SRC_LINK_TBL_LEN_INCLUDES_EXTENT) && + (edesc->desc.hdr & DESC_HDR_MODE0_ENCRYPT) == 0) { + link_tbl_ptr->j_extent = 0; + link_tbl_ptr++; + link_tbl_ptr->j_extent = DESC_PTR_LNKTBL_RETURN; + link_tbl_ptr->len = cpu_to_be16(authsize); + sg = sg_last(areq->src, edesc->src_nents ? : 1); + link_tbl_ptr->ptr = cpu_to_be32( + (char *)sg_dma_address(sg) + + sg->length - authsize); + } } else { /* Only one segment now, so no link tbl needed */ desc->ptr[4].ptr = cpu_to_be32(sg_dma_address(areq->src)); @@ -944,12 +970,11 @@ static int ipsec_esp(struct ipsec_esp_edesc *edesc, struct aead_request *areq, desc->ptr[5].ptr = cpu_to_be32(sg_dma_address(areq->dst)); } else { struct talitos_ptr *link_tbl_ptr = - &edesc->link_tbl[edesc->src_nents]; - struct scatterlist *sg; + &edesc->link_tbl[edesc->src_nents + 1]; desc->ptr[5].ptr = cpu_to_be32((struct talitos_ptr *) edesc->dma_link_tbl + - edesc->src_nents); + edesc->src_nents + 1); if (areq->src == areq->dst) { memcpy(link_tbl_ptr, &edesc->link_tbl[0], edesc->src_nents * sizeof(struct talitos_ptr)); @@ -957,14 +982,10 @@ static int ipsec_esp(struct ipsec_esp_edesc *edesc, struct aead_request *areq, sg_count = sg_to_link_tbl(areq->dst, sg_count, cryptlen, link_tbl_ptr); } + /* Add an entry to the link table for ICV data */ link_tbl_ptr += sg_count - 1; - - /* handle case where sg_last contains the ICV exclusively */ - sg = sg_last(areq->dst, edesc->dst_nents); - if (sg->length == ctx->authsize) - link_tbl_ptr--; - link_tbl_ptr->j_extent = 0; + sg_count++; link_tbl_ptr++; link_tbl_ptr->j_extent = DESC_PTR_LNKTBL_RETURN; link_tbl_ptr->len = cpu_to_be16(authsize); @@ -973,7 +994,7 @@ static int ipsec_esp(struct ipsec_esp_edesc *edesc, struct aead_request *areq, link_tbl_ptr->ptr = cpu_to_be32((struct talitos_ptr *) edesc->dma_link_tbl + edesc->src_nents + - edesc->dst_nents + 1); + edesc->dst_nents + 2); desc->ptr[5].j_extent |= DESC_PTR_LNKTBL_JUMP; dma_sync_single_for_device(ctx->dev, edesc->dma_link_tbl, @@ -1040,12 +1061,12 @@ static struct ipsec_esp_edesc *ipsec_esp_edesc_alloc(struct aead_request *areq, /* * allocate space for base edesc plus the link tables, - * allowing for a separate entry for the generated ICV (+ 1), + * allowing for two separate entries for ICV and generated ICV (+ 2), * and the ICV data itself */ alloc_len = sizeof(struct ipsec_esp_edesc); if (src_nents || dst_nents) { - dma_len = (src_nents + dst_nents + 1) * + dma_len = (src_nents + dst_nents + 2) * sizeof(struct talitos_ptr) + ctx->authsize; alloc_len += dma_len; } else { @@ -1104,7 +1125,7 @@ static int aead_authenc_decrypt(struct aead_request *req) /* stash incoming ICV for later cmp with ICV generated by the h/w */ if (edesc->dma_len) icvdata = &edesc->link_tbl[edesc->src_nents + - edesc->dst_nents + 1]; + edesc->dst_nents + 2]; else icvdata = &edesc->link_tbl[0]; @@ -1480,6 +1501,9 @@ static int talitos_probe(struct of_device *ofdev, goto err_out; } + if (of_device_is_compatible(np, "fsl,sec3.0")) + priv->features |= TALITOS_FTR_SRC_LINK_TBL_LEN_INCLUDES_EXTENT; + priv->head_lock = kmalloc(sizeof(spinlock_t) * priv->num_channels, GFP_KERNEL); priv->tail_lock = kmalloc(sizeof(spinlock_t) * priv->num_channels, -- cgit v1.2.3-70-g09d2 From ba95487df97f9c45cfdae2a47de56235e7a13dc5 Mon Sep 17 00:00:00 2001 From: Kim Phillips Date: Sun, 14 Sep 2008 13:41:19 -0700 Subject: crypto: talitos - Avoid consecutive packets going out with same IV The SEC's h/w IV out implementation DMAs the trailing encrypted payload block of the last encryption to ctx->iv. Since the last encryption may still be pending completion, we can sufficiently prevent successive packets from being transmitted with the same IV by xoring with sequence number. Also initialize alg_list earlier to prevent oopsing on a failed probe. Signed-off-by: Kim Phillips Signed-off-by: Lee Nipper Signed-off-by: Herbert Xu --- drivers/crypto/talitos.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) (limited to 'drivers/crypto/talitos.c') diff --git a/drivers/crypto/talitos.c b/drivers/crypto/talitos.c index ee827a7f7c6..b6ad3ac5916 100644 --- a/drivers/crypto/talitos.c +++ b/drivers/crypto/talitos.c @@ -1157,6 +1157,8 @@ static int aead_authenc_givencrypt( edesc->desc.hdr = ctx->desc_hdr_template | DESC_HDR_MODE0_ENCRYPT; memcpy(req->giv, ctx->iv, crypto_aead_ivsize(authenc)); + /* avoid consecutive packets going out with same IV */ + *(__be64 *)req->giv ^= cpu_to_be64(req->seq); return ipsec_esp(edesc, areq, req->giv, req->seq, ipsec_esp_encrypt_done); @@ -1449,6 +1451,8 @@ static int talitos_probe(struct of_device *ofdev, priv->ofdev = ofdev; + INIT_LIST_HEAD(&priv->alg_list); + tasklet_init(&priv->done_task, talitos_done, (unsigned long)dev); tasklet_init(&priv->error_task, talitos_error, (unsigned long)dev); @@ -1575,8 +1579,6 @@ static int talitos_probe(struct of_device *ofdev, } /* register crypto algorithms the device supports */ - INIT_LIST_HEAD(&priv->alg_list); - for (i = 0; i < ARRAY_SIZE(driver_algs); i++) { if (hw_supports(dev, driver_algs[i].desc_hdr_template)) { struct talitos_crypto_alg *t_alg; -- cgit v1.2.3-70-g09d2