From 016af9b5c51e58ecda573f14dabe85a67363b20f Mon Sep 17 00:00:00 2001 From: Joel Fernandes Date: Sun, 18 Aug 2013 00:56:11 -0500 Subject: crypto: omap-aes - Add useful debug macros When DEBUG is enabled, these macros can be used to print variables in integer and hex format, and clearly display which registers, offsets and values are being read/written , including printing the names of the offsets and their values. Using statement expression macros in read path as, Suggested-by: Joe Perches Signed-off-by: Joel Fernandes Signed-off-by: Herbert Xu --- drivers/crypto/omap-aes.c | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) (limited to 'drivers/crypto/omap-aes.c') diff --git a/drivers/crypto/omap-aes.c b/drivers/crypto/omap-aes.c index 5f798058685..e5b2120a12b 100644 --- a/drivers/crypto/omap-aes.c +++ b/drivers/crypto/omap-aes.c @@ -13,7 +13,9 @@ * */ -#define pr_fmt(fmt) "%s: " fmt, __func__ +#define pr_fmt(fmt) "%20s: " fmt, __func__ +#define prn(num) pr_debug(#num "=%d\n", num) +#define prx(num) pr_debug(#num "=%x\n", num) #include #include @@ -172,16 +174,36 @@ struct omap_aes_dev { static LIST_HEAD(dev_list); static DEFINE_SPINLOCK(list_lock); +#ifdef DEBUG +#define omap_aes_read(dd, offset) \ +({ \ + int _read_ret; \ + _read_ret = __raw_readl(dd->io_base + offset); \ + pr_debug("omap_aes_read(" #offset "=%#x)= %#x\n", \ + offset, _read_ret); \ + _read_ret; \ +}) +#else static inline u32 omap_aes_read(struct omap_aes_dev *dd, u32 offset) { return __raw_readl(dd->io_base + offset); } +#endif +#ifdef DEBUG +#define omap_aes_write(dd, offset, value) \ + do { \ + pr_debug("omap_aes_write(" #offset "=%#x) value=%#x\n", \ + offset, value); \ + __raw_writel(value, dd->io_base + offset); \ + } while (0) +#else static inline void omap_aes_write(struct omap_aes_dev *dd, u32 offset, u32 value) { __raw_writel(value, dd->io_base + offset); } +#endif static inline void omap_aes_write_mask(struct omap_aes_dev *dd, u32 offset, u32 value, u32 mask) -- cgit v1.2.3-70-g09d2 From e77c756eca76077032db23d8d4b85dcd743742b4 Mon Sep 17 00:00:00 2001 From: Joel Fernandes Date: Sat, 17 Aug 2013 21:42:24 -0500 Subject: crypto: omap-aes - Populate number of SG elements Crypto layer only passes nbytes but number of SG elements is needed for mapping or unmapping SGs at one time using dma_map* API and also needed to pass in for dmaengine prep function. We call function added to scatterwalk for this purpose in omap_aes_handle_queue to populate the values which are used later. Signed-off-by: Joel Fernandes Signed-off-by: Herbert Xu --- drivers/crypto/omap-aes.c | 6 ++++++ 1 file changed, 6 insertions(+) (limited to 'drivers/crypto/omap-aes.c') diff --git a/drivers/crypto/omap-aes.c b/drivers/crypto/omap-aes.c index e5b2120a12b..1cad12ef4f5 100644 --- a/drivers/crypto/omap-aes.c +++ b/drivers/crypto/omap-aes.c @@ -165,6 +165,8 @@ struct omap_aes_dev { void *buf_out; int dma_out; struct dma_chan *dma_lch_out; + int in_sg_len; + int out_sg_len; dma_addr_t dma_addr_out; const struct omap_aes_pdata *pdata; @@ -725,6 +727,10 @@ static int omap_aes_handle_queue(struct omap_aes_dev *dd, dd->out_offset = 0; dd->out_sg = req->dst; + dd->in_sg_len = scatterwalk_bytes_sglen(dd->in_sg, dd->total); + dd->out_sg_len = scatterwalk_bytes_sglen(dd->out_sg, dd->total); + BUG_ON(dd->in_sg_len < 0 || dd->out_sg_len < 0); + rctx = ablkcipher_request_ctx(req); ctx = crypto_ablkcipher_ctx(crypto_ablkcipher_reqtfm(req)); rctx->mode &= FLAGS_MODE_MASK; -- cgit v1.2.3-70-g09d2 From 4b645c9465065bb3f8fb717789e864aa6d675052 Mon Sep 17 00:00:00 2001 From: Joel Fernandes Date: Sat, 17 Aug 2013 21:42:25 -0500 Subject: crypto: omap-aes - Simplify DMA usage by using direct SGs In early version of this driver, assumptions were made such as DMA layer requires contiguous buffers etc. Due to this, new buffers were allocated, mapped and used for DMA. These assumptions are no longer true and DMAEngine scatter-gather DMA doesn't have such requirements. We simply the DMA operations by directly using the scatter-gather buffers provided by the crypto layer instead of creating our own. Lot of logic that handled DMA'ing only X number of bytes of the total, or as much as fitted into a 3rd party buffer is removed and is no longer required. Also, good performance improvement of atleast ~20% seen with encrypting a buffer size of 8K (1800 ops/sec vs 1400 ops/sec). Improvement will be higher for much larger blocks though such benchmarking is left as an exercise for the reader. Also DMA usage is much more simplified and coherent with rest of the code. Signed-off-by: Joel Fernandes Signed-off-by: Herbert Xu --- drivers/crypto/omap-aes.c | 147 ++++++++-------------------------------------- 1 file changed, 25 insertions(+), 122 deletions(-) (limited to 'drivers/crypto/omap-aes.c') diff --git a/drivers/crypto/omap-aes.c b/drivers/crypto/omap-aes.c index 1cad12ef4f5..114c55a59ef 100644 --- a/drivers/crypto/omap-aes.c +++ b/drivers/crypto/omap-aes.c @@ -475,22 +475,14 @@ static int sg_copy(struct scatterlist **sg, size_t *offset, void *buf, } static int omap_aes_crypt_dma(struct crypto_tfm *tfm, - struct scatterlist *in_sg, struct scatterlist *out_sg) + struct scatterlist *in_sg, struct scatterlist *out_sg, + int in_sg_len, int out_sg_len) { struct omap_aes_ctx *ctx = crypto_tfm_ctx(tfm); struct omap_aes_dev *dd = ctx->dd; struct dma_async_tx_descriptor *tx_in, *tx_out; struct dma_slave_config cfg; - dma_addr_t dma_addr_in = sg_dma_address(in_sg); - int ret, length = sg_dma_len(in_sg); - - pr_debug("len: %d\n", length); - - dd->dma_size = length; - - if (!(dd->flags & FLAGS_FAST)) - dma_sync_single_for_device(dd->dev, dma_addr_in, length, - DMA_TO_DEVICE); + int ret; memset(&cfg, 0, sizeof(cfg)); @@ -509,7 +501,7 @@ static int omap_aes_crypt_dma(struct crypto_tfm *tfm, return ret; } - tx_in = dmaengine_prep_slave_sg(dd->dma_lch_in, in_sg, 1, + tx_in = dmaengine_prep_slave_sg(dd->dma_lch_in, in_sg, in_sg_len, DMA_MEM_TO_DEV, DMA_PREP_INTERRUPT | DMA_CTRL_ACK); if (!tx_in) { @@ -528,7 +520,7 @@ static int omap_aes_crypt_dma(struct crypto_tfm *tfm, return ret; } - tx_out = dmaengine_prep_slave_sg(dd->dma_lch_out, out_sg, 1, + tx_out = dmaengine_prep_slave_sg(dd->dma_lch_out, out_sg, out_sg_len, DMA_DEV_TO_MEM, DMA_PREP_INTERRUPT | DMA_CTRL_ACK); if (!tx_out) { @@ -546,7 +538,7 @@ static int omap_aes_crypt_dma(struct crypto_tfm *tfm, dma_async_issue_pending(dd->dma_lch_out); /* start DMA */ - dd->pdata->trigger(dd, length); + dd->pdata->trigger(dd, dd->total); return 0; } @@ -555,93 +547,28 @@ static int omap_aes_crypt_dma_start(struct omap_aes_dev *dd) { struct crypto_tfm *tfm = crypto_ablkcipher_tfm( crypto_ablkcipher_reqtfm(dd->req)); - int err, fast = 0, in, out; - size_t count; - dma_addr_t addr_in, addr_out; - struct scatterlist *in_sg, *out_sg; - int len32; + int err; pr_debug("total: %d\n", dd->total); - if (sg_is_last(dd->in_sg) && sg_is_last(dd->out_sg)) { - /* check for alignment */ - in = IS_ALIGNED((u32)dd->in_sg->offset, sizeof(u32)); - out = IS_ALIGNED((u32)dd->out_sg->offset, sizeof(u32)); - - fast = in && out; + err = dma_map_sg(dd->dev, dd->in_sg, dd->in_sg_len, DMA_TO_DEVICE); + if (!err) { + dev_err(dd->dev, "dma_map_sg() error\n"); + return -EINVAL; } - if (fast) { - count = min(dd->total, sg_dma_len(dd->in_sg)); - count = min(count, sg_dma_len(dd->out_sg)); - - if (count != dd->total) { - pr_err("request length != buffer length\n"); - return -EINVAL; - } - - pr_debug("fast\n"); - - err = dma_map_sg(dd->dev, dd->in_sg, 1, DMA_TO_DEVICE); - if (!err) { - dev_err(dd->dev, "dma_map_sg() error\n"); - return -EINVAL; - } - - err = dma_map_sg(dd->dev, dd->out_sg, 1, DMA_FROM_DEVICE); - if (!err) { - dev_err(dd->dev, "dma_map_sg() error\n"); - dma_unmap_sg(dd->dev, dd->in_sg, 1, DMA_TO_DEVICE); - return -EINVAL; - } - - addr_in = sg_dma_address(dd->in_sg); - addr_out = sg_dma_address(dd->out_sg); - - in_sg = dd->in_sg; - out_sg = dd->out_sg; - - dd->flags |= FLAGS_FAST; - - } else { - /* use cache buffers */ - count = sg_copy(&dd->in_sg, &dd->in_offset, dd->buf_in, - dd->buflen, dd->total, 0); - - len32 = DIV_ROUND_UP(count, DMA_MIN) * DMA_MIN; - - /* - * The data going into the AES module has been copied - * to a local buffer and the data coming out will go - * into a local buffer so set up local SG entries for - * both. - */ - sg_init_table(&dd->in_sgl, 1); - dd->in_sgl.offset = dd->in_offset; - sg_dma_len(&dd->in_sgl) = len32; - sg_dma_address(&dd->in_sgl) = dd->dma_addr_in; - - sg_init_table(&dd->out_sgl, 1); - dd->out_sgl.offset = dd->out_offset; - sg_dma_len(&dd->out_sgl) = len32; - sg_dma_address(&dd->out_sgl) = dd->dma_addr_out; - - in_sg = &dd->in_sgl; - out_sg = &dd->out_sgl; - - addr_in = dd->dma_addr_in; - addr_out = dd->dma_addr_out; - - dd->flags &= ~FLAGS_FAST; - + err = dma_map_sg(dd->dev, dd->out_sg, dd->out_sg_len, DMA_FROM_DEVICE); + if (!err) { + dev_err(dd->dev, "dma_map_sg() error\n"); + return -EINVAL; } - dd->total -= count; - - err = omap_aes_crypt_dma(tfm, in_sg, out_sg); + err = omap_aes_crypt_dma(tfm, dd->in_sg, dd->out_sg, dd->in_sg_len, + dd->out_sg_len); if (err) { - dma_unmap_sg(dd->dev, dd->in_sg, 1, DMA_TO_DEVICE); - dma_unmap_sg(dd->dev, dd->out_sg, 1, DMA_TO_DEVICE); + dma_unmap_sg(dd->dev, dd->in_sg, dd->in_sg_len, DMA_TO_DEVICE); + dma_unmap_sg(dd->dev, dd->out_sg, dd->out_sg_len, + DMA_FROM_DEVICE); } return err; @@ -661,7 +588,6 @@ static void omap_aes_finish_req(struct omap_aes_dev *dd, int err) static int omap_aes_crypt_dma_stop(struct omap_aes_dev *dd) { int err = 0; - size_t count; pr_debug("total: %d\n", dd->total); @@ -670,21 +596,8 @@ static int omap_aes_crypt_dma_stop(struct omap_aes_dev *dd) dmaengine_terminate_all(dd->dma_lch_in); dmaengine_terminate_all(dd->dma_lch_out); - if (dd->flags & FLAGS_FAST) { - dma_unmap_sg(dd->dev, dd->out_sg, 1, DMA_FROM_DEVICE); - dma_unmap_sg(dd->dev, dd->in_sg, 1, DMA_TO_DEVICE); - } else { - dma_sync_single_for_device(dd->dev, dd->dma_addr_out, - dd->dma_size, DMA_FROM_DEVICE); - - /* copy data */ - count = sg_copy(&dd->out_sg, &dd->out_offset, dd->buf_out, - dd->buflen, dd->dma_size, 1); - if (count != dd->dma_size) { - err = -EINVAL; - pr_err("not all data converted: %u\n", count); - } - } + dma_unmap_sg(dd->dev, dd->in_sg, dd->in_sg_len, DMA_TO_DEVICE); + dma_unmap_sg(dd->dev, dd->out_sg, dd->out_sg_len, DMA_FROM_DEVICE); return err; } @@ -754,21 +667,11 @@ static int omap_aes_handle_queue(struct omap_aes_dev *dd, static void omap_aes_done_task(unsigned long data) { struct omap_aes_dev *dd = (struct omap_aes_dev *)data; - int err; - - pr_debug("enter\n"); - err = omap_aes_crypt_dma_stop(dd); - - err = dd->err ? : err; - - if (dd->total && !err) { - err = omap_aes_crypt_dma_start(dd); - if (!err) - return; /* DMA started. Not fininishing. */ - } + pr_debug("enter done_task\n"); - omap_aes_finish_req(dd, err); + omap_aes_crypt_dma_stop(dd); + omap_aes_finish_req(dd, 0); omap_aes_handle_queue(dd, NULL); pr_debug("exit\n"); -- cgit v1.2.3-70-g09d2 From 0a641712ef9459df5d8c19fc19e686887257b3e3 Mon Sep 17 00:00:00 2001 From: Joel Fernandes Date: Sat, 17 Aug 2013 21:42:26 -0500 Subject: crypto: omap-aes - Sync SG before DMA operation Earlier functions that did a similar sync are replaced by the dma_sync_sg_* which can operate on entire SG list. Signed-off-by: Joel Fernandes Signed-off-by: Herbert Xu --- drivers/crypto/omap-aes.c | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'drivers/crypto/omap-aes.c') diff --git a/drivers/crypto/omap-aes.c b/drivers/crypto/omap-aes.c index 114c55a59ef..4ed2a8c4039 100644 --- a/drivers/crypto/omap-aes.c +++ b/drivers/crypto/omap-aes.c @@ -484,6 +484,8 @@ static int omap_aes_crypt_dma(struct crypto_tfm *tfm, struct dma_slave_config cfg; int ret; + dma_sync_sg_for_device(dd->dev, dd->in_sg, in_sg_len, DMA_TO_DEVICE); + memset(&cfg, 0, sizeof(cfg)); cfg.src_addr = dd->phys_base + AES_REG_DATA_N(dd, 0); @@ -670,6 +672,8 @@ static void omap_aes_done_task(unsigned long data) pr_debug("enter done_task\n"); + dma_sync_sg_for_cpu(dd->dev, dd->in_sg, dd->in_sg_len, DMA_FROM_DEVICE); + omap_aes_crypt_dma_stop(dd); omap_aes_finish_req(dd, 0); omap_aes_handle_queue(dd, NULL); -- cgit v1.2.3-70-g09d2 From 0c063a22d50606e5e405e954be2de3810372dc27 Mon Sep 17 00:00:00 2001 From: Joel Fernandes Date: Sat, 17 Aug 2013 21:42:27 -0500 Subject: crypto: omap-aes - Remove previously used intermediate buffers Intermdiate buffers were allocated, mapped and used for DMA. These are no longer required as we use the SGs from crypto layer directly in previous commits in the series. Also along with it, remove the logic for copying SGs etc as they are no longer used, and all the associated variables in omap_aes_device. Signed-off-by: Joel Fernandes Signed-off-by: Herbert Xu --- drivers/crypto/omap-aes.c | 90 ----------------------------------------------- 1 file changed, 90 deletions(-) (limited to 'drivers/crypto/omap-aes.c') diff --git a/drivers/crypto/omap-aes.c b/drivers/crypto/omap-aes.c index 4ed2a8c4039..81f0e848b8d 100644 --- a/drivers/crypto/omap-aes.c +++ b/drivers/crypto/omap-aes.c @@ -150,25 +150,13 @@ struct omap_aes_dev { struct ablkcipher_request *req; size_t total; struct scatterlist *in_sg; - struct scatterlist in_sgl; - size_t in_offset; struct scatterlist *out_sg; - struct scatterlist out_sgl; - size_t out_offset; - - size_t buflen; - void *buf_in; - size_t dma_size; int dma_in; struct dma_chan *dma_lch_in; - dma_addr_t dma_addr_in; - void *buf_out; int dma_out; struct dma_chan *dma_lch_out; int in_sg_len; int out_sg_len; - dma_addr_t dma_addr_out; - const struct omap_aes_pdata *pdata; }; @@ -347,33 +335,6 @@ static int omap_aes_dma_init(struct omap_aes_dev *dd) dd->dma_lch_out = NULL; dd->dma_lch_in = NULL; - dd->buf_in = (void *)__get_free_pages(GFP_KERNEL, OMAP_AES_CACHE_SIZE); - dd->buf_out = (void *)__get_free_pages(GFP_KERNEL, OMAP_AES_CACHE_SIZE); - dd->buflen = PAGE_SIZE << OMAP_AES_CACHE_SIZE; - dd->buflen &= ~(AES_BLOCK_SIZE - 1); - - if (!dd->buf_in || !dd->buf_out) { - dev_err(dd->dev, "unable to alloc pages.\n"); - goto err_alloc; - } - - /* MAP here */ - dd->dma_addr_in = dma_map_single(dd->dev, dd->buf_in, dd->buflen, - DMA_TO_DEVICE); - if (dma_mapping_error(dd->dev, dd->dma_addr_in)) { - dev_err(dd->dev, "dma %d bytes error\n", dd->buflen); - err = -EINVAL; - goto err_map_in; - } - - dd->dma_addr_out = dma_map_single(dd->dev, dd->buf_out, dd->buflen, - DMA_FROM_DEVICE); - if (dma_mapping_error(dd->dev, dd->dma_addr_out)) { - dev_err(dd->dev, "dma %d bytes error\n", dd->buflen); - err = -EINVAL; - goto err_map_out; - } - dma_cap_zero(mask); dma_cap_set(DMA_SLAVE, mask); @@ -400,14 +361,6 @@ static int omap_aes_dma_init(struct omap_aes_dev *dd) err_dma_out: dma_release_channel(dd->dma_lch_in); err_dma_in: - dma_unmap_single(dd->dev, dd->dma_addr_out, dd->buflen, - DMA_FROM_DEVICE); -err_map_out: - dma_unmap_single(dd->dev, dd->dma_addr_in, dd->buflen, DMA_TO_DEVICE); -err_map_in: - free_pages((unsigned long)dd->buf_out, OMAP_AES_CACHE_SIZE); - free_pages((unsigned long)dd->buf_in, OMAP_AES_CACHE_SIZE); -err_alloc: if (err) pr_err("error: %d\n", err); return err; @@ -417,11 +370,6 @@ static void omap_aes_dma_cleanup(struct omap_aes_dev *dd) { dma_release_channel(dd->dma_lch_out); dma_release_channel(dd->dma_lch_in); - dma_unmap_single(dd->dev, dd->dma_addr_out, dd->buflen, - DMA_FROM_DEVICE); - dma_unmap_single(dd->dev, dd->dma_addr_in, dd->buflen, DMA_TO_DEVICE); - free_pages((unsigned long)dd->buf_out, OMAP_AES_CACHE_SIZE); - free_pages((unsigned long)dd->buf_in, OMAP_AES_CACHE_SIZE); } static void sg_copy_buf(void *buf, struct scatterlist *sg, @@ -438,42 +386,6 @@ static void sg_copy_buf(void *buf, struct scatterlist *sg, scatterwalk_done(&walk, out, 0); } -static int sg_copy(struct scatterlist **sg, size_t *offset, void *buf, - size_t buflen, size_t total, int out) -{ - unsigned int count, off = 0; - - while (buflen && total) { - count = min((*sg)->length - *offset, total); - count = min(count, buflen); - - if (!count) - return off; - - /* - * buflen and total are AES_BLOCK_SIZE size aligned, - * so count should be also aligned - */ - - sg_copy_buf(buf + off, *sg, *offset, count, out); - - off += count; - buflen -= count; - *offset += count; - total -= count; - - if (*offset == (*sg)->length) { - *sg = sg_next(*sg); - if (*sg) - *offset = 0; - else - total = 0; - } - } - - return off; -} - static int omap_aes_crypt_dma(struct crypto_tfm *tfm, struct scatterlist *in_sg, struct scatterlist *out_sg, int in_sg_len, int out_sg_len) @@ -637,9 +549,7 @@ static int omap_aes_handle_queue(struct omap_aes_dev *dd, /* assign new request to device */ dd->req = req; dd->total = req->nbytes; - dd->in_offset = 0; dd->in_sg = req->src; - dd->out_offset = 0; dd->out_sg = req->dst; dd->in_sg_len = scatterwalk_bytes_sglen(dd->in_sg, dd->total); -- cgit v1.2.3-70-g09d2 From 67216756ea8b9bfa738e0f0a263094f459ab1a12 Mon Sep 17 00:00:00 2001 From: Joel Fernandes Date: Sat, 17 Aug 2013 21:42:28 -0500 Subject: crypto: omap-aes - Add IRQ info and helper macros Add IRQ information to pdata and helper macros. These are required for PIO-mode support. Signed-off-by: Joel Fernandes Signed-off-by: Herbert Xu --- drivers/crypto/omap-aes.c | 8 ++++++++ 1 file changed, 8 insertions(+) (limited to 'drivers/crypto/omap-aes.c') diff --git a/drivers/crypto/omap-aes.c b/drivers/crypto/omap-aes.c index 81f0e848b8d..68c44253137 100644 --- a/drivers/crypto/omap-aes.c +++ b/drivers/crypto/omap-aes.c @@ -76,6 +76,10 @@ #define AES_REG_LENGTH_N(x) (0x54 + ((x) * 0x04)) +#define AES_REG_IRQ_STATUS(dd) ((dd)->pdata->irq_status_ofs) +#define AES_REG_IRQ_ENABLE(dd) ((dd)->pdata->irq_enable_ofs) +#define AES_REG_IRQ_DATA_IN BIT(1) +#define AES_REG_IRQ_DATA_OUT BIT(2) #define DEFAULT_TIMEOUT (5*HZ) #define FLAGS_MODE_MASK 0x000f @@ -121,6 +125,8 @@ struct omap_aes_pdata { u32 data_ofs; u32 rev_ofs; u32 mask_ofs; + u32 irq_enable_ofs; + u32 irq_status_ofs; u32 dma_enable_in; u32 dma_enable_out; @@ -847,6 +853,8 @@ static const struct omap_aes_pdata omap_aes_pdata_omap4 = { .data_ofs = 0x60, .rev_ofs = 0x80, .mask_ofs = 0x84, + .irq_status_ofs = 0x8c, + .irq_enable_ofs = 0x90, .dma_enable_in = BIT(5), .dma_enable_out = BIT(6), .major_mask = 0x0700, -- cgit v1.2.3-70-g09d2 From 1bf95cca8f407cc0d6f21708fdbb17d1dd531bec Mon Sep 17 00:00:00 2001 From: Joel Fernandes Date: Sat, 17 Aug 2013 21:42:29 -0500 Subject: crypto: omap-aes - PIO mode: Add IRQ handler and walk SGs We add an IRQ handler that implements a state-machine for PIO-mode and data structures for walking the scatter-gather list. The IRQ handler is called in succession both when data is available to read or next data can be sent for processing. This process continues till the entire in/out SG lists have been walked. Once the SG-list has been completely walked, the IRQ handler schedules the done_task tasklet. Also add a useful macro that is used through out the IRQ code for a common pattern of calculating how much an SG list has been walked. This improves code readability and avoids checkpatch errors. Signed-off-by: Joel Fernandes Signed-off-by: Herbert Xu --- drivers/crypto/omap-aes.c | 90 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 90 insertions(+) (limited to 'drivers/crypto/omap-aes.c') diff --git a/drivers/crypto/omap-aes.c b/drivers/crypto/omap-aes.c index 68c44253137..9909f93255b 100644 --- a/drivers/crypto/omap-aes.c +++ b/drivers/crypto/omap-aes.c @@ -40,6 +40,8 @@ #define DST_MAXBURST 4 #define DMA_MIN (DST_MAXBURST * sizeof(u32)) +#define _calc_walked(inout) (dd->inout##_walk.offset - dd->inout##_sg->offset) + /* OMAP TRM gives bitfields as start:end, where start is the higher bit number. For example 7:0 */ #define FLD_MASK(start, end) (((1 << ((start) - (end) + 1)) - 1) << (end)) @@ -92,6 +94,8 @@ #define FLAGS_FAST BIT(5) #define FLAGS_BUSY BIT(6) +#define AES_BLOCK_WORDS (AES_BLOCK_SIZE >> 2) + struct omap_aes_ctx { struct omap_aes_dev *dd; @@ -157,6 +161,8 @@ struct omap_aes_dev { size_t total; struct scatterlist *in_sg; struct scatterlist *out_sg; + struct scatter_walk in_walk; + struct scatter_walk out_walk; int dma_in; struct dma_chan *dma_lch_in; int dma_out; @@ -863,6 +869,90 @@ static const struct omap_aes_pdata omap_aes_pdata_omap4 = { .minor_shift = 0, }; +static irqreturn_t omap_aes_irq(int irq, void *dev_id) +{ + struct omap_aes_dev *dd = dev_id; + u32 status, i; + u32 *src, *dst; + + status = omap_aes_read(dd, AES_REG_IRQ_STATUS(dd)); + if (status & AES_REG_IRQ_DATA_IN) { + omap_aes_write(dd, AES_REG_IRQ_ENABLE(dd), 0x0); + + BUG_ON(!dd->in_sg); + + BUG_ON(_calc_walked(in) > dd->in_sg->length); + + src = sg_virt(dd->in_sg) + _calc_walked(in); + + for (i = 0; i < AES_BLOCK_WORDS; i++) { + omap_aes_write(dd, AES_REG_DATA_N(dd, i), *src); + + scatterwalk_advance(&dd->in_walk, 4); + if (dd->in_sg->length == _calc_walked(in)) { + dd->in_sg = scatterwalk_sg_next(dd->in_sg); + if (dd->in_sg) { + scatterwalk_start(&dd->in_walk, + dd->in_sg); + src = sg_virt(dd->in_sg) + + _calc_walked(in); + } + } else { + src++; + } + } + + /* Clear IRQ status */ + status &= ~AES_REG_IRQ_DATA_IN; + omap_aes_write(dd, AES_REG_IRQ_STATUS(dd), status); + + /* Enable DATA_OUT interrupt */ + omap_aes_write(dd, AES_REG_IRQ_ENABLE(dd), 0x4); + + } else if (status & AES_REG_IRQ_DATA_OUT) { + omap_aes_write(dd, AES_REG_IRQ_ENABLE(dd), 0x0); + + BUG_ON(!dd->out_sg); + + BUG_ON(_calc_walked(out) > dd->out_sg->length); + + dst = sg_virt(dd->out_sg) + _calc_walked(out); + + for (i = 0; i < AES_BLOCK_WORDS; i++) { + *dst = omap_aes_read(dd, AES_REG_DATA_N(dd, i)); + scatterwalk_advance(&dd->out_walk, 4); + if (dd->out_sg->length == _calc_walked(out)) { + dd->out_sg = scatterwalk_sg_next(dd->out_sg); + if (dd->out_sg) { + scatterwalk_start(&dd->out_walk, + dd->out_sg); + dst = sg_virt(dd->out_sg) + + _calc_walked(out); + } + } else { + dst++; + } + } + + dd->total -= AES_BLOCK_SIZE; + + BUG_ON(dd->total < 0); + + /* Clear IRQ status */ + status &= ~AES_REG_IRQ_DATA_OUT; + omap_aes_write(dd, AES_REG_IRQ_STATUS(dd), status); + + if (!dd->total) + /* All bytes read! */ + tasklet_schedule(&dd->done_task); + else + /* Enable DATA_IN interrupt for next block */ + omap_aes_write(dd, AES_REG_IRQ_ENABLE(dd), 0x2); + } + + return IRQ_HANDLED; +} + static const struct of_device_id omap_aes_of_match[] = { { .compatible = "ti,omap2-aes", -- cgit v1.2.3-70-g09d2 From 98837abc86ebd26c1518c91cc5e2a9344837e6a8 Mon Sep 17 00:00:00 2001 From: Joel Fernandes Date: Sat, 17 Aug 2013 21:42:30 -0500 Subject: crypto: omap-aes - PIO mode: platform data for OMAP4/AM437x and trigger We initialize the scatter gather walk lists needed for PIO mode and avoid all DMA paths such as mapping/unmapping buffers by checking for the pio_only flag. Signed-off-by: Joel Fernandes Signed-off-by: Herbert Xu --- drivers/crypto/omap-aes.c | 43 ++++++++++++++++++++++++++++++------------- 1 file changed, 30 insertions(+), 13 deletions(-) (limited to 'drivers/crypto/omap-aes.c') diff --git a/drivers/crypto/omap-aes.c b/drivers/crypto/omap-aes.c index 9909f93255b..685535e4c78 100644 --- a/drivers/crypto/omap-aes.c +++ b/drivers/crypto/omap-aes.c @@ -169,6 +169,7 @@ struct omap_aes_dev { struct dma_chan *dma_lch_out; int in_sg_len; int out_sg_len; + int pio_only; const struct omap_aes_pdata *pdata; }; @@ -408,6 +409,16 @@ static int omap_aes_crypt_dma(struct crypto_tfm *tfm, struct dma_slave_config cfg; int ret; + if (dd->pio_only) { + scatterwalk_start(&dd->in_walk, dd->in_sg); + scatterwalk_start(&dd->out_walk, dd->out_sg); + + /* Enable DATAIN interrupt and let it take + care of the rest */ + omap_aes_write(dd, AES_REG_IRQ_ENABLE(dd), 0x2); + return 0; + } + dma_sync_sg_for_device(dd->dev, dd->in_sg, in_sg_len, DMA_TO_DEVICE); memset(&cfg, 0, sizeof(cfg)); @@ -477,21 +488,25 @@ static int omap_aes_crypt_dma_start(struct omap_aes_dev *dd) pr_debug("total: %d\n", dd->total); - err = dma_map_sg(dd->dev, dd->in_sg, dd->in_sg_len, DMA_TO_DEVICE); - if (!err) { - dev_err(dd->dev, "dma_map_sg() error\n"); - return -EINVAL; - } + if (!dd->pio_only) { + err = dma_map_sg(dd->dev, dd->in_sg, dd->in_sg_len, + DMA_TO_DEVICE); + if (!err) { + dev_err(dd->dev, "dma_map_sg() error\n"); + return -EINVAL; + } - err = dma_map_sg(dd->dev, dd->out_sg, dd->out_sg_len, DMA_FROM_DEVICE); - if (!err) { - dev_err(dd->dev, "dma_map_sg() error\n"); - return -EINVAL; + err = dma_map_sg(dd->dev, dd->out_sg, dd->out_sg_len, + DMA_FROM_DEVICE); + if (!err) { + dev_err(dd->dev, "dma_map_sg() error\n"); + return -EINVAL; + } } err = omap_aes_crypt_dma(tfm, dd->in_sg, dd->out_sg, dd->in_sg_len, dd->out_sg_len); - if (err) { + if (err && !dd->pio_only) { dma_unmap_sg(dd->dev, dd->in_sg, dd->in_sg_len, DMA_TO_DEVICE); dma_unmap_sg(dd->dev, dd->out_sg, dd->out_sg_len, DMA_FROM_DEVICE); @@ -594,9 +609,11 @@ static void omap_aes_done_task(unsigned long data) pr_debug("enter done_task\n"); - dma_sync_sg_for_cpu(dd->dev, dd->in_sg, dd->in_sg_len, DMA_FROM_DEVICE); - - omap_aes_crypt_dma_stop(dd); + if (!dd->pio_only) { + dma_sync_sg_for_device(dd->dev, dd->out_sg, dd->out_sg_len, + DMA_FROM_DEVICE); + omap_aes_crypt_dma_stop(dd); + } omap_aes_finish_req(dd, 0); omap_aes_handle_queue(dd, NULL); -- cgit v1.2.3-70-g09d2 From 1801ad9483b796271a985ab71cfe26f7bbeae6dd Mon Sep 17 00:00:00 2001 From: Joel Fernandes Date: Sat, 17 Aug 2013 21:42:31 -0500 Subject: crypto: omap-aes - Switch to PIO mode during probe In cases where requesting for DMA channels fails for some reason, or channel numbers are not provided in DT or platform data, we switch to PIO-only mode also checking if platform provides IRQ numbers and interrupt register offsets in DT and platform data. All dma-only paths are avoided in this mode. Signed-off-by: Joel Fernandes Signed-off-by: Herbert Xu --- drivers/crypto/omap-aes.c | 28 +++++++++++++++++++++++----- 1 file changed, 23 insertions(+), 5 deletions(-) (limited to 'drivers/crypto/omap-aes.c') diff --git a/drivers/crypto/omap-aes.c b/drivers/crypto/omap-aes.c index 685535e4c78..7a08a152838 100644 --- a/drivers/crypto/omap-aes.c +++ b/drivers/crypto/omap-aes.c @@ -1075,7 +1075,7 @@ static int omap_aes_probe(struct platform_device *pdev) struct omap_aes_dev *dd; struct crypto_alg *algp; struct resource res; - int err = -ENOMEM, i, j; + int err = -ENOMEM, i, j, irq = -1; u32 reg; dd = kzalloc(sizeof(struct omap_aes_dev), GFP_KERNEL); @@ -1118,8 +1118,23 @@ static int omap_aes_probe(struct platform_device *pdev) tasklet_init(&dd->queue_task, omap_aes_queue_task, (unsigned long)dd); err = omap_aes_dma_init(dd); - if (err) - goto err_dma; + if (err && AES_REG_IRQ_STATUS(dd) && AES_REG_IRQ_ENABLE(dd)) { + dd->pio_only = 1; + + irq = platform_get_irq(pdev, 0); + if (irq < 0) { + dev_err(dev, "can't get IRQ resource\n"); + goto err_irq; + } + + err = request_irq(irq, omap_aes_irq, 0, + dev_name(dev), dd); + if (err) { + dev_err(dev, "Unable to grab omap-aes IRQ\n"); + goto err_irq; + } + } + INIT_LIST_HEAD(&dd->list); spin_lock(&list_lock); @@ -1147,8 +1162,11 @@ err_algs: for (j = dd->pdata->algs_info[i].registered - 1; j >= 0; j--) crypto_unregister_alg( &dd->pdata->algs_info[i].algs_list[j]); - omap_aes_dma_cleanup(dd); -err_dma: + if (dd->pio_only) + free_irq(irq, dd); + else + omap_aes_dma_cleanup(dd); +err_irq: tasklet_kill(&dd->done_task); tasklet_kill(&dd->queue_task); pm_runtime_disable(dev); -- cgit v1.2.3-70-g09d2 From 6242332ff2f3431c4fb6f4b21581f38f16569c13 Mon Sep 17 00:00:00 2001 From: Joel Fernandes Date: Sat, 17 Aug 2013 21:42:32 -0500 Subject: crypto: omap-aes - Add support for cases of unaligned lengths For cases where offset/length of on any page of the input SG is not aligned by AES_BLOCK_SIZE, we copy all the pages from the input SG list into a contiguous buffer and prepare a single element SG list for this buffer with length as the total bytes to crypt. This is requried for cases such as when an SG list of 16 bytes total size contains 16 pages each containing 1 byte. DMA using the direct buffers of such instances is not possible. For this purpose, we first detect if the unaligned case and accordingly allocate enough number of pages to satisfy the request and prepare SG lists. We then copy data into the buffer, and copy data out of it on completion. Signed-off-by: Joel Fernandes Signed-off-by: Herbert Xu --- drivers/crypto/omap-aes.c | 86 +++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 83 insertions(+), 3 deletions(-) (limited to 'drivers/crypto/omap-aes.c') diff --git a/drivers/crypto/omap-aes.c b/drivers/crypto/omap-aes.c index 7a08a152838..2fd22ca6a58 100644 --- a/drivers/crypto/omap-aes.c +++ b/drivers/crypto/omap-aes.c @@ -158,9 +158,23 @@ struct omap_aes_dev { struct tasklet_struct queue_task; struct ablkcipher_request *req; + + /* + * total is used by PIO mode for book keeping so introduce + * variable total_save as need it to calc page_order + */ size_t total; + size_t total_save; + struct scatterlist *in_sg; struct scatterlist *out_sg; + + /* Buffers for copying for unaligned cases */ + struct scatterlist in_sgl; + struct scatterlist out_sgl; + struct scatterlist *orig_out; + int sgs_copied; + struct scatter_walk in_walk; struct scatter_walk out_walk; int dma_in; @@ -537,12 +551,51 @@ static int omap_aes_crypt_dma_stop(struct omap_aes_dev *dd) dmaengine_terminate_all(dd->dma_lch_in); dmaengine_terminate_all(dd->dma_lch_out); - dma_unmap_sg(dd->dev, dd->in_sg, dd->in_sg_len, DMA_TO_DEVICE); - dma_unmap_sg(dd->dev, dd->out_sg, dd->out_sg_len, DMA_FROM_DEVICE); - return err; } +int omap_aes_check_aligned(struct scatterlist *sg) +{ + while (sg) { + if (!IS_ALIGNED(sg->offset, 4)) + return -1; + if (!IS_ALIGNED(sg->length, AES_BLOCK_SIZE)) + return -1; + sg = sg_next(sg); + } + return 0; +} + +int omap_aes_copy_sgs(struct omap_aes_dev *dd) +{ + void *buf_in, *buf_out; + int pages; + + pages = get_order(dd->total); + + buf_in = (void *)__get_free_pages(GFP_ATOMIC, pages); + buf_out = (void *)__get_free_pages(GFP_ATOMIC, pages); + + if (!buf_in || !buf_out) { + pr_err("Couldn't allocated pages for unaligned cases.\n"); + return -1; + } + + dd->orig_out = dd->out_sg; + + sg_copy_buf(buf_in, dd->in_sg, 0, dd->total, 0); + + sg_init_table(&dd->in_sgl, 1); + sg_set_buf(&dd->in_sgl, buf_in, dd->total); + dd->in_sg = &dd->in_sgl; + + sg_init_table(&dd->out_sgl, 1); + sg_set_buf(&dd->out_sgl, buf_out, dd->total); + dd->out_sg = &dd->out_sgl; + + return 0; +} + static int omap_aes_handle_queue(struct omap_aes_dev *dd, struct ablkcipher_request *req) { @@ -576,9 +629,19 @@ static int omap_aes_handle_queue(struct omap_aes_dev *dd, /* assign new request to device */ dd->req = req; dd->total = req->nbytes; + dd->total_save = req->nbytes; dd->in_sg = req->src; dd->out_sg = req->dst; + if (omap_aes_check_aligned(dd->in_sg) || + omap_aes_check_aligned(dd->out_sg)) { + if (omap_aes_copy_sgs(dd)) + pr_err("Failed to copy SGs for unaligned cases\n"); + dd->sgs_copied = 1; + } else { + dd->sgs_copied = 0; + } + dd->in_sg_len = scatterwalk_bytes_sglen(dd->in_sg, dd->total); dd->out_sg_len = scatterwalk_bytes_sglen(dd->out_sg, dd->total); BUG_ON(dd->in_sg_len < 0 || dd->out_sg_len < 0); @@ -606,14 +669,31 @@ static int omap_aes_handle_queue(struct omap_aes_dev *dd, static void omap_aes_done_task(unsigned long data) { struct omap_aes_dev *dd = (struct omap_aes_dev *)data; + void *buf_in, *buf_out; + int pages; pr_debug("enter done_task\n"); if (!dd->pio_only) { dma_sync_sg_for_device(dd->dev, dd->out_sg, dd->out_sg_len, DMA_FROM_DEVICE); + dma_unmap_sg(dd->dev, dd->in_sg, dd->in_sg_len, DMA_TO_DEVICE); + dma_unmap_sg(dd->dev, dd->out_sg, dd->out_sg_len, + DMA_FROM_DEVICE); omap_aes_crypt_dma_stop(dd); } + + if (dd->sgs_copied) { + buf_in = sg_virt(&dd->in_sgl); + buf_out = sg_virt(&dd->out_sgl); + + sg_copy_buf(buf_out, dd->orig_out, 0, dd->total_save, 1); + + pages = get_order(dd->total_save); + free_pages((unsigned long)buf_in, pages); + free_pages((unsigned long)buf_out, pages); + } + omap_aes_finish_req(dd, 0); omap_aes_handle_queue(dd, NULL); -- cgit v1.2.3-70-g09d2 From 05007c10ff76a9d6d85c007fe3a7f531611fda5c Mon Sep 17 00:00:00 2001 From: Joel Fernandes Date: Sat, 17 Aug 2013 21:42:33 -0500 Subject: crypto: omap-aes - Convert kzalloc to devm_kzalloc Use devm_kzalloc instead of kzalloc. With this change, there is no need to call kfree in error/exit paths. Signed-off-by: Joel Fernandes Signed-off-by: Herbert Xu --- drivers/crypto/omap-aes.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) (limited to 'drivers/crypto/omap-aes.c') diff --git a/drivers/crypto/omap-aes.c b/drivers/crypto/omap-aes.c index 2fd22ca6a58..1f3d816cc9e 100644 --- a/drivers/crypto/omap-aes.c +++ b/drivers/crypto/omap-aes.c @@ -1158,7 +1158,7 @@ static int omap_aes_probe(struct platform_device *pdev) int err = -ENOMEM, i, j, irq = -1; u32 reg; - dd = kzalloc(sizeof(struct omap_aes_dev), GFP_KERNEL); + dd = devm_kzalloc(dev, sizeof(struct omap_aes_dev), GFP_KERNEL); if (dd == NULL) { dev_err(dev, "unable to alloc data struct.\n"); goto err_data; @@ -1251,7 +1251,6 @@ err_irq: tasklet_kill(&dd->queue_task); pm_runtime_disable(dev); err_res: - kfree(dd); dd = NULL; err_data: dev_err(dev, "initialization failed.\n"); @@ -1279,7 +1278,6 @@ static int omap_aes_remove(struct platform_device *pdev) tasklet_kill(&dd->queue_task); omap_aes_dma_cleanup(dd); pm_runtime_disable(dd->dev); - kfree(dd); dd = NULL; return 0; -- cgit v1.2.3-70-g09d2 From bce2a22885e657099d9f2cf82269bb4629ba744d Mon Sep 17 00:00:00 2001 From: Joel Fernandes Date: Sat, 17 Aug 2013 21:42:34 -0500 Subject: crypto: omap-aes - Convert request_irq to devm_request_irq Keeps request_irq exit/error code paths simpler. Suggested-by: Lokesh Vutla Signed-off-by: Joel Fernandes Signed-off-by: Herbert Xu --- drivers/crypto/omap-aes.c | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) (limited to 'drivers/crypto/omap-aes.c') diff --git a/drivers/crypto/omap-aes.c b/drivers/crypto/omap-aes.c index 1f3d816cc9e..ce791c2f81f 100644 --- a/drivers/crypto/omap-aes.c +++ b/drivers/crypto/omap-aes.c @@ -1207,7 +1207,7 @@ static int omap_aes_probe(struct platform_device *pdev) goto err_irq; } - err = request_irq(irq, omap_aes_irq, 0, + err = devm_request_irq(dev, irq, omap_aes_irq, 0, dev_name(dev), dd); if (err) { dev_err(dev, "Unable to grab omap-aes IRQ\n"); @@ -1242,9 +1242,7 @@ err_algs: for (j = dd->pdata->algs_info[i].registered - 1; j >= 0; j--) crypto_unregister_alg( &dd->pdata->algs_info[i].algs_list[j]); - if (dd->pio_only) - free_irq(irq, dd); - else + if (!dd->pio_only) omap_aes_dma_cleanup(dd); err_irq: tasklet_kill(&dd->done_task); -- cgit v1.2.3-70-g09d2