diff options
Diffstat (limited to 'drivers/dma')
| -rw-r--r-- | drivers/dma/Kconfig | 3 | ||||
| -rw-r--r-- | drivers/dma/amba-pl08x.c | 640 | ||||
| -rw-r--r-- | drivers/dma/at_hdmac.c | 164 | ||||
| -rw-r--r-- | drivers/dma/at_hdmac_regs.h | 24 | ||||
| -rw-r--r-- | drivers/dma/dmatest.c | 23 | ||||
| -rw-r--r-- | drivers/dma/dw_dmac.c | 5 | ||||
| -rw-r--r-- | drivers/dma/ep93xx_dma.c | 1 | ||||
| -rw-r--r-- | drivers/dma/imx-dma.c | 2 | ||||
| -rw-r--r-- | drivers/dma/imx-sdma.c | 49 | ||||
| -rw-r--r-- | drivers/dma/intel_mid_dma.c | 10 | ||||
| -rw-r--r-- | drivers/dma/ipu/ipu_idmac.c | 66 | ||||
| -rw-r--r-- | drivers/dma/mpc512x_dma.c | 1 | ||||
| -rw-r--r-- | drivers/dma/mxs-dma.c | 45 | ||||
| -rw-r--r-- | drivers/dma/pch_dma.c | 7 | ||||
| -rw-r--r-- | drivers/dma/pl330.c | 231 | ||||
| -rw-r--r-- | drivers/dma/shdma.c | 129 | ||||
| -rw-r--r-- | drivers/dma/shdma.h | 7 | ||||
| -rw-r--r-- | drivers/dma/ste_dma40.c | 1 | ||||
| -rw-r--r-- | drivers/dma/timb_dma.c | 5 |
19 files changed, 911 insertions, 502 deletions
diff --git a/drivers/dma/Kconfig b/drivers/dma/Kconfig index 2e3b3d38c46..ab8f469f5cf 100644 --- a/drivers/dma/Kconfig +++ b/drivers/dma/Kconfig @@ -193,7 +193,8 @@ config ARCH_HAS_ASYNC_TX_FIND_CHANNEL config PL330_DMA tristate "DMA API Driver for PL330" select DMA_ENGINE - depends on PL330 + depends on ARM_AMBA + select PL330 help Select if your platform has one or more PL330 DMACs. You need to provide platform specific settings via diff --git a/drivers/dma/amba-pl08x.c b/drivers/dma/amba-pl08x.c index be21e3f138a..b7cbd1ab1db 100644 --- a/drivers/dma/amba-pl08x.c +++ b/drivers/dma/amba-pl08x.c @@ -66,32 +66,29 @@ * after the final transfer signalled by LBREQ or LSREQ. The DMAC * will then move to the next LLI entry. * - * Only the former works sanely with scatter lists, so we only implement - * the DMAC flow control method. However, peripherals which use the LBREQ - * and LSREQ signals (eg, MMCI) are unable to use this mode, which through - * these hardware restrictions prevents them from using scatter DMA. - * * Global TODO: * - Break out common code from arch/arm/mach-s3c64xx and share */ -#include <linux/device.h> -#include <linux/init.h> -#include <linux/module.h> -#include <linux/interrupt.h> -#include <linux/slab.h> -#include <linux/delay.h> -#include <linux/dma-mapping.h> -#include <linux/dmapool.h> -#include <linux/dmaengine.h> #include <linux/amba/bus.h> #include <linux/amba/pl08x.h> #include <linux/debugfs.h> +#include <linux/delay.h> +#include <linux/device.h> +#include <linux/dmaengine.h> +#include <linux/dmapool.h> +#include <linux/dma-mapping.h> +#include <linux/init.h> +#include <linux/interrupt.h> +#include <linux/module.h> +#include <linux/pm_runtime.h> #include <linux/seq_file.h> - +#include <linux/slab.h> #include <asm/hardware/pl080.h> #define DRIVER_NAME "pl08xdmac" +static struct amba_driver pl08x_amba_driver; + /** * struct vendor_data - vendor-specific config parameters for PL08x derivatives * @channels: the number of channels available in this variant @@ -126,7 +123,8 @@ struct pl08x_lli { * @phy_chans: array of data for the physical channels * @pool: a pool for the LLI descriptors * @pool_ctr: counter of LLIs in the pool - * @lli_buses: bitmask to or in to LLI pointer selecting AHB port for LLI fetches + * @lli_buses: bitmask to or in to LLI pointer selecting AHB port for LLI + * fetches * @mem_buses: set to indicate memory transfers on AHB2. * @lock: a spinlock for this struct */ @@ -149,14 +147,6 @@ struct pl08x_driver_data { * PL08X specific defines */ -/* - * Memory boundaries: the manual for PL08x says that the controller - * cannot read past a 1KiB boundary, so these defines are used to - * create transfer LLIs that do not cross such boundaries. - */ -#define PL08X_BOUNDARY_SHIFT (10) /* 1KB 0x400 */ -#define PL08X_BOUNDARY_SIZE (1 << PL08X_BOUNDARY_SHIFT) - /* Size (bytes) of each LLI buffer allocated for one transfer */ # define PL08X_LLI_TSFR_SIZE 0x2000 @@ -272,7 +262,6 @@ static void pl08x_resume_phy_chan(struct pl08x_phy_chan *ch) writel(val, ch->base + PL080_CH_CONFIG); } - /* * pl08x_terminate_phy_chan() stops the channel, clears the FIFO and * clears any pending interrupt status. This should not be used for @@ -363,7 +352,9 @@ static u32 pl08x_getbytes_chan(struct pl08x_dma_chan *plchan) if (!list_empty(&plchan->pend_list)) { struct pl08x_txd *txdi; list_for_each_entry(txdi, &plchan->pend_list, node) { - bytes += txdi->len; + struct pl08x_sg *dsg; + list_for_each_entry(dsg, &txd->dsg_list, node) + bytes += dsg->len; } } @@ -407,6 +398,7 @@ pl08x_get_phy_channel(struct pl08x_driver_data *pl08x, return NULL; } + pm_runtime_get_sync(&pl08x->adev->dev); return ch; } @@ -420,6 +412,8 @@ static inline void pl08x_put_phy_channel(struct pl08x_driver_data *pl08x, /* Stop the channel and clear its interrupts */ pl08x_terminate_phy_chan(pl08x, ch); + pm_runtime_put(&pl08x->adev->dev); + /* Mark it as free */ ch->serving = NULL; spin_unlock_irqrestore(&ch->lock, flags); @@ -499,36 +493,30 @@ struct pl08x_lli_build_data { }; /* - * Autoselect a master bus to use for the transfer this prefers the - * destination bus if both available if fixed address on one bus the - * other will be chosen + * Autoselect a master bus to use for the transfer. Slave will be the chosen as + * victim in case src & dest are not similarly aligned. i.e. If after aligning + * masters address with width requirements of transfer (by sending few byte by + * byte data), slave is still not aligned, then its width will be reduced to + * BYTE. + * - prefers the destination bus if both available + * - prefers bus with fixed address (i.e. peripheral) */ static void pl08x_choose_master_bus(struct pl08x_lli_build_data *bd, struct pl08x_bus_data **mbus, struct pl08x_bus_data **sbus, u32 cctl) { if (!(cctl & PL080_CONTROL_DST_INCR)) { - *mbus = &bd->srcbus; - *sbus = &bd->dstbus; - } else if (!(cctl & PL080_CONTROL_SRC_INCR)) { *mbus = &bd->dstbus; *sbus = &bd->srcbus; + } else if (!(cctl & PL080_CONTROL_SRC_INCR)) { + *mbus = &bd->srcbus; + *sbus = &bd->dstbus; } else { - if (bd->dstbus.buswidth == 4) { + if (bd->dstbus.buswidth >= bd->srcbus.buswidth) { *mbus = &bd->dstbus; *sbus = &bd->srcbus; - } else if (bd->srcbus.buswidth == 4) { - *mbus = &bd->srcbus; - *sbus = &bd->dstbus; - } else if (bd->dstbus.buswidth == 2) { - *mbus = &bd->dstbus; - *sbus = &bd->srcbus; - } else if (bd->srcbus.buswidth == 2) { + } else { *mbus = &bd->srcbus; *sbus = &bd->dstbus; - } else { - /* bd->srcbus.buswidth == 1 */ - *mbus = &bd->dstbus; - *sbus = &bd->srcbus; } } } @@ -547,7 +535,8 @@ static void pl08x_fill_lli_for_desc(struct pl08x_lli_build_data *bd, llis_va[num_llis].cctl = cctl; llis_va[num_llis].src = bd->srcbus.addr; llis_va[num_llis].dst = bd->dstbus.addr; - llis_va[num_llis].lli = llis_bus + (num_llis + 1) * sizeof(struct pl08x_lli); + llis_va[num_llis].lli = llis_bus + (num_llis + 1) * + sizeof(struct pl08x_lli); llis_va[num_llis].lli |= bd->lli_bus; if (cctl & PL080_CONTROL_SRC_INCR) @@ -560,16 +549,12 @@ static void pl08x_fill_lli_for_desc(struct pl08x_lli_build_data *bd, bd->remainder -= len; } -/* - * Return number of bytes to fill to boundary, or len. - * This calculation works for any value of addr. - */ -static inline size_t pl08x_pre_boundary(u32 addr, size_t len) +static inline void prep_byte_width_lli(struct pl08x_lli_build_data *bd, + u32 *cctl, u32 len, int num_llis, size_t *total_bytes) { - size_t boundary_len = PL08X_BOUNDARY_SIZE - - (addr & (PL08X_BOUNDARY_SIZE - 1)); - - return min(boundary_len, len); + *cctl = pl08x_cctl_bits(*cctl, 1, 1, len); + pl08x_fill_lli_for_desc(bd, num_llis, len, *cctl); + (*total_bytes) += len; } /* @@ -583,13 +568,12 @@ static int pl08x_fill_llis_for_desc(struct pl08x_driver_data *pl08x, struct pl08x_bus_data *mbus, *sbus; struct pl08x_lli_build_data bd; int num_llis = 0; - u32 cctl; - size_t max_bytes_per_lli; - size_t total_bytes = 0; + u32 cctl, early_bytes = 0; + size_t max_bytes_per_lli, total_bytes; struct pl08x_lli *llis_va; + struct pl08x_sg *dsg; - txd->llis_va = dma_pool_alloc(pl08x->pool, GFP_NOWAIT, - &txd->llis_bus); + txd->llis_va = dma_pool_alloc(pl08x->pool, GFP_NOWAIT, &txd->llis_bus); if (!txd->llis_va) { dev_err(&pl08x->adev->dev, "%s no memory for llis\n", __func__); return 0; @@ -597,13 +581,9 @@ static int pl08x_fill_llis_for_desc(struct pl08x_driver_data *pl08x, pl08x->pool_ctr++; - /* Get the default CCTL */ - cctl = txd->cctl; - bd.txd = txd; - bd.srcbus.addr = txd->src_addr; - bd.dstbus.addr = txd->dst_addr; bd.lli_bus = (pl08x->lli_buses & PL08X_AHB2) ? PL080_LLI_LM_AHB2 : 0; + cctl = txd->cctl; /* Find maximum width of the source bus */ bd.srcbus.maxwidth = @@ -615,215 +595,179 @@ static int pl08x_fill_llis_for_desc(struct pl08x_driver_data *pl08x, pl08x_get_bytes_for_cctl((cctl & PL080_CONTROL_DWIDTH_MASK) >> PL080_CONTROL_DWIDTH_SHIFT); - /* Set up the bus widths to the maximum */ - bd.srcbus.buswidth = bd.srcbus.maxwidth; - bd.dstbus.buswidth = bd.dstbus.maxwidth; + list_for_each_entry(dsg, &txd->dsg_list, node) { + total_bytes = 0; + cctl = txd->cctl; - /* - * Bytes transferred == tsize * MIN(buswidths), not max(buswidths) - */ - max_bytes_per_lli = min(bd.srcbus.buswidth, bd.dstbus.buswidth) * - PL080_CONTROL_TRANSFER_SIZE_MASK; + bd.srcbus.addr = dsg->src_addr; + bd.dstbus.addr = dsg->dst_addr; + bd.remainder = dsg->len; + bd.srcbus.buswidth = bd.srcbus.maxwidth; + bd.dstbus.buswidth = bd.dstbus.maxwidth; - /* We need to count this down to zero */ - bd.remainder = txd->len; + pl08x_choose_master_bus(&bd, &mbus, &sbus, cctl); - /* - * Choose bus to align to - * - prefers destination bus if both available - * - if fixed address on one bus chooses other - */ - pl08x_choose_master_bus(&bd, &mbus, &sbus, cctl); - - dev_vdbg(&pl08x->adev->dev, "src=0x%08x%s/%u dst=0x%08x%s/%u len=%zu llimax=%zu\n", - bd.srcbus.addr, cctl & PL080_CONTROL_SRC_INCR ? "+" : "", - bd.srcbus.buswidth, - bd.dstbus.addr, cctl & PL080_CONTROL_DST_INCR ? "+" : "", - bd.dstbus.buswidth, - bd.remainder, max_bytes_per_lli); - dev_vdbg(&pl08x->adev->dev, "mbus=%s sbus=%s\n", - mbus == &bd.srcbus ? "src" : "dst", - sbus == &bd.srcbus ? "src" : "dst"); - - if (txd->len < mbus->buswidth) { - /* Less than a bus width available - send as single bytes */ - while (bd.remainder) { - dev_vdbg(&pl08x->adev->dev, - "%s single byte LLIs for a transfer of " - "less than a bus width (remain 0x%08x)\n", - __func__, bd.remainder); - cctl = pl08x_cctl_bits(cctl, 1, 1, 1); - pl08x_fill_lli_for_desc(&bd, num_llis++, 1, cctl); - total_bytes++; - } - } else { - /* Make one byte LLIs until master bus is aligned */ - while ((mbus->addr) % (mbus->buswidth)) { - dev_vdbg(&pl08x->adev->dev, - "%s adjustment lli for less than bus width " - "(remain 0x%08x)\n", - __func__, bd.remainder); - cctl = pl08x_cctl_bits(cctl, 1, 1, 1); - pl08x_fill_lli_for_desc(&bd, num_llis++, 1, cctl); - total_bytes++; - } + dev_vdbg(&pl08x->adev->dev, "src=0x%08x%s/%u dst=0x%08x%s/%u len=%zu\n", + bd.srcbus.addr, cctl & PL080_CONTROL_SRC_INCR ? "+" : "", + bd.srcbus.buswidth, + bd.dstbus.addr, cctl & PL080_CONTROL_DST_INCR ? "+" : "", + bd.dstbus.buswidth, + bd.remainder); + dev_vdbg(&pl08x->adev->dev, "mbus=%s sbus=%s\n", + mbus == &bd.srcbus ? "src" : "dst", + sbus == &bd.srcbus ? "src" : "dst"); /* - * Master now aligned - * - if slave is not then we must set its width down + * Zero length is only allowed if all these requirements are + * met: + * - flow controller is peripheral. + * - src.addr is aligned to src.width + * - dst.addr is aligned to dst.width + * + * sg_len == 1 should be true, as there can be two cases here: + * + * - Memory addresses are contiguous and are not scattered. + * Here, Only one sg will be passed by user driver, with + * memory address and zero length. We pass this to controller + * and after the transfer it will receive the last burst + * request from peripheral and so transfer finishes. + * + * - Memory addresses are scattered and are not contiguous. + * Here, Obviously as DMA controller doesn't know when a lli's + * transfer gets over, it can't load next lli. So in this + * case, there has to be an assumption that only one lli is + * supported. Thus, we can't have scattered addresses. */ - if (sbus->addr % sbus->buswidth) { - dev_dbg(&pl08x->adev->dev, - "%s set down bus width to one byte\n", - __func__); + if (!bd.remainder) { + u32 fc = (txd->ccfg & PL080_CONFIG_FLOW_CONTROL_MASK) >> + PL080_CONFIG_FLOW_CONTROL_SHIFT; + if (!((fc >= PL080_FLOW_SRC2DST_DST) && + (fc <= PL080_FLOW_SRC2DST_SRC))) { + dev_err(&pl08x->adev->dev, "%s sg len can't be zero", + __func__); + return 0; + } + + if ((bd.srcbus.addr % bd.srcbus.buswidth) || + (bd.srcbus.addr % bd.srcbus.buswidth)) { + dev_err(&pl08x->adev->dev, + "%s src & dst address must be aligned to src" + " & dst width if peripheral is flow controller", + __func__); + return 0; + } - sbus->buswidth = 1; + cctl = pl08x_cctl_bits(cctl, bd.srcbus.buswidth, + bd.dstbus.buswidth, 0); + pl08x_fill_lli_for_desc(&bd, num_llis++, 0, cctl); + break; } /* - * Make largest possible LLIs until less than one bus - * width left + * Send byte by byte for following cases + * - Less than a bus width available + * - until master bus is aligned */ - while (bd.remainder > (mbus->buswidth - 1)) { - size_t lli_len, target_len, tsize, odd_bytes; + if (bd.remainder < mbus->buswidth) + early_bytes = bd.remainder; + else if ((mbus->addr) % (mbus->buswidth)) { + early_bytes = mbus->buswidth - (mbus->addr) % + (mbus->buswidth); + if ((bd.remainder - early_bytes) < mbus->buswidth) + early_bytes = bd.remainder; + } + if (early_bytes) { + dev_vdbg(&pl08x->adev->dev, + "%s byte width LLIs (remain 0x%08x)\n", + __func__, bd.remainder); + prep_byte_width_lli(&bd, &cctl, early_bytes, num_llis++, + &total_bytes); + } + + if (bd.remainder) { /* - * If enough left try to send max possible, - * otherwise try to send the remainder + * Master now aligned + * - if slave is not then we must set its width down */ - target_len = min(bd.remainder, max_bytes_per_lli); + if (sbus->addr % sbus->buswidth) { + dev_dbg(&pl08x->adev->dev, + "%s set down bus width to one byte\n", + __func__); + + sbus->buswidth = 1; + } /* - * Set bus lengths for incrementing buses to the - * number of bytes which fill to next memory boundary, - * limiting on the target length calculated above. + * Bytes transferred = tsize * src width, not + * MIN(buswidths) */ - if (cctl & PL080_CONTROL_SRC_INCR) - bd.srcbus.fill_bytes = - pl08x_pre_boundary(bd.srcbus.addr, - target_len); - else - bd.srcbus.fill_bytes = target_len; - - if (cctl & PL080_CONTROL_DST_INCR) - bd.dstbus.fill_bytes = - pl08x_pre_boundary(bd.dstbus.addr, - target_len); - else - bd.dstbus.fill_bytes = target_len; - - /* Find the nearest */ - lli_len = min(bd.srcbus.fill_bytes, - bd.dstbus.fill_bytes); - - BUG_ON(lli_len > bd.remainder); - - if (lli_len <= 0) { - dev_err(&pl08x->adev->dev, - "%s lli_len is %zu, <= 0\n", - __func__, lli_len); - return 0; - } + max_bytes_per_lli = bd.srcbus.buswidth * + PL080_CONTROL_TRANSFER_SIZE_MASK; + dev_vdbg(&pl08x->adev->dev, + "%s max bytes per lli = %zu\n", + __func__, max_bytes_per_lli); + + /* + * Make largest possible LLIs until less than one bus + * width left + */ + while (bd.remainder > (mbus->buswidth - 1)) { + size_t lli_len, tsize, width; - if (lli_len == target_len) { - /* - * Can send what we wanted. - * Maintain alignment - */ - lli_len = (lli_len/mbus->buswidth) * - mbus->buswidth; - odd_bytes = 0; - } else { /* - * So now we know how many bytes to transfer - * to get to the nearest boundary. The next - * LLI will past the boundary. However, we - * may be working to a boundary on the slave - * bus. We need to ensure the master stays - * aligned, and that we are working in - * multiples of the bus widths. + * If enough left try to send max possible, + * otherwise try to send the remainder */ - odd_bytes = lli_len % mbus->buswidth; - lli_len -= odd_bytes; - - } + lli_len = min(bd.remainder, max_bytes_per_lli); - if (lli_len) { /* - * Check against minimum bus alignment: - * Calculate actual transfer size in relation - * to bus width an get a maximum remainder of - * the smallest bus width - 1 + * Check against maximum bus alignment: + * Calculate actual transfer size in relation to + * bus width an get a maximum remainder of the + * highest bus width - 1 */ - /* FIXME: use round_down()? */ - tsize = lli_len / min(mbus->buswidth, - sbus->buswidth); - lli_len = tsize * min(mbus->buswidth, - sbus->buswidth); - - if (target_len != lli_len) { - dev_vdbg(&pl08x->adev->dev, - "%s can't send what we want. Desired 0x%08zx, lli of 0x%08zx bytes in txd of 0x%08zx\n", - __func__, target_len, lli_len, txd->len); - } - - cctl = pl08x_cctl_bits(cctl, - bd.srcbus.buswidth, - bd.dstbus.buswidth, - tsize); + width = max(mbus->buswidth, sbus->buswidth); + lli_len = (lli_len / width) * width; + tsize = lli_len / bd.srcbus.buswidth; dev_vdbg(&pl08x->adev->dev, - "%s fill lli with single lli chunk of size 0x%08zx (remainder 0x%08zx)\n", + "%s fill lli with single lli chunk of " + "size 0x%08zx (remainder 0x%08zx)\n", __func__, lli_len, bd.remainder); + + cctl = pl08x_cctl_bits(cctl, bd.srcbus.buswidth, + bd.dstbus.buswidth, tsize); pl08x_fill_lli_for_desc(&bd, num_llis++, - lli_len, cctl); + lli_len, cctl); total_bytes += lli_len; } - - if (odd_bytes) { - /* - * Creep past the boundary, maintaining - * master alignment - */ - int j; - for (j = 0; (j < mbus->buswidth) - && (bd.remainder); j++) { - cctl = pl08x_cctl_bits(cctl, 1, 1, 1); - dev_vdbg(&pl08x->adev->dev, - "%s align with boundary, single byte (remain 0x%08zx)\n", - __func__, bd.remainder); - pl08x_fill_lli_for_desc(&bd, - num_llis++, 1, cctl); - total_bytes++; - } + /* + * Send any odd bytes + */ + if (bd.remainder) { + dev_vdbg(&pl08x->adev->dev, + "%s align with boundary, send odd bytes (remain %zu)\n", + __func__, bd.remainder); + prep_byte_width_lli(&bd, &cctl, bd.remainder, + num_llis++, &total_bytes); } } - /* - * Send any odd bytes - */ - while (bd.remainder) { - cctl = pl08x_cctl_bits(cctl, 1, 1, 1); - dev_vdbg(&pl08x->adev->dev, - "%s align with boundary, single odd byte (remain %zu)\n", - __func__, bd.remainder); - pl08x_fill_lli_for_desc(&bd, num_llis++, 1, cctl); - total_bytes++; + if (total_bytes != dsg->len) { + dev_err(&pl08x->adev->dev, + "%s size of encoded lli:s don't match total txd, transferred 0x%08zx from size 0x%08zx\n", + __func__, total_bytes, dsg->len); + return 0; } - } - if (total_bytes != txd->len) { - dev_err(&pl08x->adev->dev, - "%s size of encoded lli:s don't match total txd, transferred 0x%08zx from size 0x%08zx\n", - __func__, total_bytes, txd->len); - return 0; - } - if (num_llis >= MAX_NUM_TSFR_LLIS) { - dev_err(&pl08x->adev->dev, - "%s need to increase MAX_NUM_TSFR_LLIS from 0x%08x\n", - __func__, (u32) MAX_NUM_TSFR_LLIS); - return 0; + if (num_llis >= MAX_NUM_TSFR_LLIS) { + dev_err(&pl08x->adev->dev, + "%s need to increase MAX_NUM_TSFR_LLIS from 0x%08x\n", + __func__, (u32) MAX_NUM_TSFR_LLIS); + return 0; + } } llis_va = txd->llis_va; @@ -856,11 +800,19 @@ static int pl08x_fill_llis_for_desc(struct pl08x_driver_data *pl08x, static void pl08x_free_txd(struct pl08x_driver_data *pl08x, struct pl08x_txd *txd) { + struct pl08x_sg *dsg, *_dsg; + /* Free the LLI */ - dma_pool_free(pl08x->pool, txd->llis_va, txd->llis_bus); + if (txd->llis_va) + dma_pool_free(pl08x->pool, txd->llis_va, txd->llis_bus); pl08x->pool_ctr--; + list_for_each_entry_safe(dsg, _dsg, &txd->dsg_list, node) { + list_del(&dsg->node); + kfree(dsg); + } + kfree(txd); } @@ -917,9 +869,7 @@ static int prep_phy_channel(struct pl08x_dma_chan *plchan, * need, but for slaves the physical signals may be muxed! * Can the platform allow us to use this channel? */ - if (plchan->slave && - ch->signal < 0 && - pl08x->pd->get_signal) { + if (plchan->slave && pl08x->pd->get_signal) { ret = pl08x->pd->get_signal(plchan); if (ret < 0) { dev_dbg(&pl08x->adev->dev, @@ -1008,10 +958,8 @@ static struct dma_async_tx_descriptor *pl08x_prep_dma_interrupt( * If slaves are relying on interrupts to signal completion this function * must not be called with interrupts disabled. */ -static enum dma_status -pl08x_dma_tx_status(struct dma_chan *chan, - dma_cookie_t cookie, - struct dma_tx_state *txstate) +static enum dma_status pl08x_dma_tx_status(struct dma_chan *chan, + dma_cookie_t cookie, struct dma_tx_state *txstate) { struct pl08x_dma_chan *plchan = to_pl08x_chan(chan); dma_cookie_t last_used; @@ -1253,7 +1201,9 @@ static int pl08x_prep_channel_resources(struct pl08x_dma_chan *plchan, num_llis = pl08x_fill_llis_for_desc(pl08x, txd); if (!num_llis) { - kfree(txd); + spin_lock_irqsave(&plchan->lock, flags); + pl08x_free_txd(pl08x, txd); + spin_unlock_irqrestore(&plchan->lock, flags); return -EINVAL; } @@ -1301,13 +1251,14 @@ static int pl08x_prep_channel_resources(struct pl08x_dma_chan *plchan, static struct pl08x_txd *pl08x_get_txd(struct pl08x_dma_chan *plchan, unsigned long flags) { - struct pl08x_txd *txd = kzalloc(sizeof(struct pl08x_txd), GFP_NOWAIT); + struct pl08x_txd *txd = kzalloc(sizeof(*txd), GFP_NOWAIT); if (txd) { dma_async_tx_descriptor_init(&txd->tx, &plchan->chan); txd->tx.flags = flags; txd->tx.tx_submit = pl08x_tx_submit; INIT_LIST_HEAD(&txd->node); + INIT_LIST_HEAD(&txd->dsg_list); /* Always enable error and terminal interrupts */ txd->ccfg = PL080_CONFIG_ERR_IRQ_MASK | @@ -1326,6 +1277,7 @@ static struct dma_async_tx_descriptor *pl08x_prep_dma_memcpy( struct pl08x_dma_chan *plchan = to_pl08x_chan(chan); struct pl08x_driver_data *pl08x = plchan->host; struct pl08x_txd *txd; + struct pl08x_sg *dsg; int ret; txd = pl08x_get_txd(plchan, flags); @@ -1335,10 +1287,19 @@ static struct dma_async_tx_descriptor *pl08x_prep_dma_memcpy( return NULL; } + dsg = kzalloc(sizeof(struct pl08x_sg), GFP_NOWAIT); + if (!dsg) { + pl08x_free_txd(pl08x, txd); + dev_err(&pl08x->adev->dev, "%s no memory for pl080 sg\n", + __func__); + return NULL; + } + list_add_tail(&dsg->node, &txd->dsg_list); + txd->direction = DMA_NONE; - txd->src_addr = src; - txd->dst_addr = dest; - txd->len = len; + dsg->src_addr = src; + dsg->dst_addr = dest; + dsg->len = len; /* Set platform data for m2m */ txd->ccfg |= PL080_FLOW_MEM2MEM << PL080_CONFIG_FLOW_CONTROL_SHIFT; @@ -1367,19 +1328,13 @@ static struct dma_async_tx_descriptor *pl08x_prep_slave_sg( struct pl08x_dma_chan *plchan = to_pl08x_chan(chan); struct pl08x_driver_data *pl08x = plchan->host; struct pl08x_txd *txd; - int ret; - - /* - * Current implementation ASSUMES only one sg - */ - if (sg_len != 1) { - dev_err(&pl08x->adev->dev, "%s prepared too long sglist\n", - __func__); - BUG(); - } + struct pl08x_sg *dsg; + struct scatterlist *sg; + dma_addr_t slave_addr; + int ret, tmp; dev_dbg(&pl08x->adev->dev, "%s prepare transaction of %d bytes from %s\n", - __func__, sgl->length, plchan->name); + __func__, sgl->length, plchan->name); txd = pl08x_get_txd(plchan, flags); if (!txd) { @@ -1398,24 +1353,49 @@ static struct dma_async_tx_descriptor *pl08x_prep_slave_sg( * channel target address dynamically at runtime. */ txd->direction = direction; - txd->len = sgl->length; if (direction == DMA_TO_DEVICE) { - txd->ccfg |= PL080_FLOW_MEM2PER << PL080_CONFIG_FLOW_CONTROL_SHIFT; txd->cctl = plchan->dst_cctl; - txd->src_addr = sgl->dma_address; - txd->dst_addr = plchan->dst_addr; + slave_addr = plchan->dst_addr; } else if (direction == DMA_FROM_DEVICE) { - txd->ccfg |= PL080_FLOW_PER2MEM << PL080_CONFIG_FLOW_CONTROL_SHIFT; txd->cctl = plchan->src_cctl; - txd->src_addr = plchan->src_addr; - txd->dst_addr = sgl->dma_address; + slave_addr = plchan->src_addr; } else { + pl08x_free_txd(pl08x, txd); dev_err(&pl08x->adev->dev, "%s direction unsupported\n", __func__); return NULL; } + if (plchan->cd->device_fc) + tmp = (direction == DMA_TO_DEVICE) ? PL080_FLOW_MEM2PER_PER : + PL080_FLOW_PER2MEM_PER; + else + tmp = (direction == DMA_TO_DEVICE) ? PL080_FLOW_MEM2PER : + PL080_FLOW_PER2MEM; + + txd->ccfg |= tmp << PL080_CONFIG_FLOW_CONTROL_SHIFT; + + for_each_sg(sgl, sg, sg_len, tmp) { + dsg = kzalloc(sizeof(struct pl08x_sg), GFP_NOWAIT); + if (!dsg) { + pl08x_free_txd(pl08x, txd); + dev_err(&pl08x->adev->dev, "%s no mem for pl080 sg\n", + __func__); + return NULL; + } + list_add_tail(&dsg->node, &txd->dsg_list); + + dsg->len = sg_dma_len(sg); + if (direction == DMA_TO_DEVICE) { + dsg->src_addr = sg_phys(sg); + dsg->dst_addr = slave_addr; + } else { + dsg->src_addr = slave_addr; + dsg->dst_addr = sg_phys(sg); + } + } + ret = pl08x_prep_channel_resources(plchan, txd); if (ret) return NULL; @@ -1489,9 +1469,15 @@ static int pl08x_control(struct dma_chan *chan, enum dma_ctrl_cmd cmd, bool pl08x_filter_id(struct dma_chan *chan, void *chan_id) { - struct pl08x_dma_chan *plchan = to_pl08x_chan(chan); + struct pl08x_dma_chan *plchan; char *name = chan_id; + /* Reject channels for devices not bound to this driver */ + if (chan->device->dev->driver != &pl08x_amba_driver.drv) + return false; + + plchan = to_pl08x_chan(chan); + /* Check that the channel is not taken! */ if (!strcmp(plchan->name, name)) return true; @@ -1507,34 +1493,34 @@ bool pl08x_filter_id(struct dma_chan *chan, void *chan_id) */ static void pl08x_ensure_on(struct pl08x_driver_data *pl08x) { - u32 val; - - val = readl(pl08x->base + PL080_CONFIG); - val &= ~(PL080_CONFIG_M2_BE | PL080_CONFIG_M1_BE | PL080_CONFIG_ENABLE); - /* We implicitly clear bit 1 and that means little-endian mode */ - val |= PL080_CONFIG_ENABLE; - writel(val, pl08x->base + PL080_CONFIG); + writel(PL080_CONFIG_ENABLE, pl08x->base + PL080_CONFIG); } static void pl08x_unmap_buffers(struct pl08x_txd *txd) { struct device *dev = txd->tx.chan->device->dev; + struct pl08x_sg *dsg; if (!(txd->tx.flags & DMA_COMPL_SKIP_SRC_UNMAP)) { if (txd->tx.flags & DMA_COMPL_SRC_UNMAP_SINGLE) - dma_unmap_single(dev, txd->src_addr, txd->len, - DMA_TO_DEVICE); - else - dma_unmap_page(dev, txd->src_addr, txd->len, - DMA_TO_DEVICE); + list_for_each_entry(dsg, &txd->dsg_list, node) + dma_unmap_single(dev, dsg->src_addr, dsg->len, + DMA_TO_DEVICE); + else { + list_for_each_entry(dsg, &txd->dsg_list, node) + dma_unmap_page(dev, dsg->src_addr, dsg->len, + DMA_TO_DEVICE); + } } if (!(txd->tx.flags & DMA_COMPL_SKIP_DEST_UNMAP)) { if (txd->tx.flags & DMA_COMPL_DEST_UNMAP_SINGLE) - dma_unmap_single(dev, txd->dst_addr, txd->len, - DMA_FROM_DEVICE); + list_for_each_entry(dsg, &txd->dsg_list, node) + dma_unmap_single(dev, dsg->dst_addr, dsg->len, + DMA_FROM_DEVICE); else - dma_unmap_page(dev, txd->dst_addr, txd->len, - DMA_FROM_DEVICE); + list_for_each_entry(dsg, &txd->dsg_list, node) + dma_unmap_page(dev, dsg->dst_addr, dsg->len, + DMA_FROM_DEVICE); } } @@ -1589,8 +1575,8 @@ static void pl08x_tasklet(unsigned long data) */ list_for_each_entry(waiting, &pl08x->memcpy.channels, chan.device_node) { - if (waiting->state == PL08X_CHAN_WAITING && - waiting->waiting != NULL) { + if (waiting->state == PL08X_CHAN_WAITING && + waiting->waiting != NULL) { int ret; /* This should REALLY not fail now */ @@ -1630,38 +1616,40 @@ static void pl08x_tasklet(unsigned long data) static irqreturn_t pl08x_irq(int irq, void *dev) { struct pl08x_driver_data *pl08x = dev; - u32 mask = 0; - u32 val; - int i; - - val = readl(pl08x->base + PL080_ERR_STATUS); - if (val) { - /* An error interrupt (on one or more channels) */ - dev_err(&pl08x->adev->dev, - "%s error interrupt, register value 0x%08x\n", - __func__, val); - /* - * Simply clear ALL PL08X error interrupts, - * regardless of channel and cause - * FIXME: should be 0x00000003 on PL081 really. - */ - writel(0x000000FF, pl08x->base + PL080_ERR_CLEAR); + u32 mask = 0, err, tc, i; + + /* check & clear - ERR & TC interrupts */ + err = readl(pl08x->base + PL080_ERR_STATUS); + if (err) { + dev_err(&pl08x->adev->dev, "%s error interrupt, register value 0x%08x\n", + __func__, err); + writel(err, pl08x->base + PL080_ERR_CLEAR); } - val = readl(pl08x->base + PL080_INT_STATUS); + tc = readl(pl08x->base + PL080_INT_STATUS); + if (tc) + writel(tc, pl08x->base + PL080_TC_CLEAR); + + if (!err && !tc) + return IRQ_NONE; + for (i = 0; i < pl08x->vd->channels; i++) { - if ((1 << i) & val) { + if (((1 << i) & err) || ((1 << i) & tc)) { /* Locate physical channel */ struct pl08x_phy_chan *phychan = &pl08x->phy_chans[i]; struct pl08x_dma_chan *plchan = phychan->serving; + if (!plchan) { + dev_err(&pl08x->adev->dev, + "%s Error TC interrupt on unused channel: 0x%08x\n", + __func__, i); + continue; + } + /* Schedule tasklet on this channel */ tasklet_schedule(&plchan->tasklet); - mask |= (1 << i); } } - /* Clear only the terminal interrupts on channels we processed */ - writel(mask, pl08x->base + PL080_TC_CLEAR); return mask ? IRQ_HANDLED : IRQ_NONE; } @@ -1685,9 +1673,7 @@ static void pl08x_dma_slave_init(struct pl08x_dma_chan *chan) * Make a local wrapper to hold required data */ static int pl08x_dma_init_virtual_channels(struct pl08x_driver_data *pl08x, - struct dma_device *dmadev, - unsigned int channels, - bool slave) + struct dma_device *dmadev, unsigned int channels, bool slave) { struct pl08x_dma_chan *chan; int i; @@ -1700,7 +1686,7 @@ static int pl08x_dma_init_virtual_channels(struct pl08x_driver_data *pl08x, * to cope with that situation. */ for (i = 0; i < channels; i++) { - chan = kzalloc(sizeof(struct pl08x_dma_chan), GFP_KERNEL); + chan = kzalloc(sizeof(*chan), GFP_KERNEL); if (!chan) { dev_err(&pl08x->adev->dev, "%s no memory for channel\n", __func__); @@ -1728,7 +1714,7 @@ static int pl08x_dma_init_virtual_channels(struct pl08x_driver_data *pl08x, kfree(chan); continue; } - dev_info(&pl08x->adev->dev, + dev_dbg(&pl08x->adev->dev, "initialize virtual channel \"%s\"\n", chan->name); @@ -1837,9 +1823,9 @@ static const struct file_operations pl08x_debugfs_operations = { static void init_pl08x_debugfs(struct pl08x_driver_data *pl08x) { /* Expose a simple debugfs interface to view all clocks */ - (void) debugfs_create_file(dev_name(&pl08x->adev->dev), S_IFREG | S_IRUGO, - NULL, pl08x, - &pl08x_debugfs_operations); + (void) debugfs_create_file(dev_name(&pl08x->adev->dev), + S_IFREG | S_IRUGO, NULL, pl08x, + &pl08x_debugfs_operations); } #else @@ -1860,12 +1846,15 @@ static int pl08x_probe(struct amba_device *adev, const struct amba_id *id) return ret; /* Create the driver state holder */ - pl08x = kzalloc(sizeof(struct pl08x_driver_data), GFP_KERNEL); + pl08x = kzalloc(sizeof(*pl08x), GFP_KERNEL); if (!pl08x) { ret = -ENOMEM; goto out_no_pl08x; } + pm_runtime_set_active(&adev->dev); + pm_runtime_enable(&adev->dev); + /* Initialize memcpy engine */ dma_cap_set(DMA_MEMCPY, pl08x->memcpy.cap_mask); pl08x->memcpy.dev = &adev->dev; @@ -1939,7 +1928,7 @@ static int pl08x_probe(struct amba_device *adev, const struct amba_id *id) } /* Initialize physical channels */ - pl08x->phy_chans = kmalloc((vd->channels * sizeof(struct pl08x_phy_chan)), + pl08x->phy_chans = kmalloc((vd->channels * sizeof(*pl08x->phy_chans)), GFP_KERNEL); if (!pl08x->phy_chans) { dev_err(&adev->dev, "%s failed to allocate " @@ -1956,9 +1945,8 @@ static int pl08x_probe(struct amba_device *adev, const struct amba_id *id) spin_lock_init(&ch->lock); ch->serving = NULL; ch->signal = -1; - dev_info(&adev->dev, - "physical channel %d is %s\n", i, - pl08x_phy_channel_busy(ch) ? "BUSY" : "FREE"); + dev_dbg(&adev->dev, "physical channel %d is %s\n", + i, pl08x_phy_channel_busy(ch) ? "BUSY" : "FREE"); } /* Register as many memcpy channels as there are physical channels */ @@ -1974,8 +1962,7 @@ static int pl08x_probe(struct amba_device *adev, const struct amba_id *id) /* Register slave channels */ ret = pl08x_dma_init_virtual_channels(pl08x, &pl08x->slave, - pl08x->pd->num_slave_channels, - true); + pl08x->pd->num_slave_channels, true); if (ret <= 0) { dev_warn(&pl08x->adev->dev, "%s failed to enumerate slave channels - %d\n", @@ -2005,6 +1992,8 @@ static int pl08x_probe(struct amba_device *adev, const struct amba_id *id) dev_info(&pl08x->adev->dev, "DMA: PL%03x rev%u at 0x%08llx irq %d\n", amba_part(adev), amba_rev(adev), (unsigned long long)adev->res.start, adev->irq[0]); + + pm_runtime_put(&adev->dev); return 0; out_no_slave_reg: @@ -2023,6 +2012,9 @@ out_no_ioremap: dma_pool_destroy(pl08x->pool); out_no_lli_pool: out_no_platdata: + pm_runtime_put(&adev->dev); + pm_runtime_disable(&adev->dev); + kfree(pl08x); out_no_pl08x: amba_release_regions(adev); diff --git a/drivers/dma/at_hdmac.c b/drivers/dma/at_hdmac.c index 6a483eac7b3..fcfa0a8b5c5 100644 --- a/drivers/dma/at_hdmac.c +++ b/drivers/dma/at_hdmac.c @@ -107,10 +107,11 @@ static struct at_desc *atc_desc_get(struct at_dma_chan *atchan) { struct at_desc *desc, *_desc; struct at_desc *ret = NULL; + unsigned long flags; unsigned int i = 0; LIST_HEAD(tmp_list); - spin_lock_bh(&atchan->lock); + spin_lock_irqsave(&atchan->lock, flags); list_for_each_entry_safe(desc, _desc, &atchan->free_list, desc_node) { i++; if (async_tx_test_ack(&desc->txd)) { @@ -121,7 +122,7 @@ static struct at_desc *atc_desc_get(struct at_dma_chan *atchan) dev_dbg(chan2dev(&atchan->chan_common), "desc %p not ACKed\n", desc); } - spin_unlock_bh(&atchan->lock); + spin_unlock_irqrestore(&atchan->lock, flags); dev_vdbg(chan2dev(&atchan->chan_common), "scanned %u descriptors on freelist\n", i); @@ -129,9 +130,9 @@ static struct at_desc *atc_desc_get(struct at_dma_chan *atchan) if (!ret) { ret = atc_alloc_descriptor(&atchan->chan_common, GFP_ATOMIC); if (ret) { - spin_lock_bh(&atchan->lock); + spin_lock_irqsave(&atchan->lock, flags); atchan->descs_allocated++; - spin_unlock_bh(&atchan->lock); + spin_unlock_irqrestore(&atchan->lock, flags); } else { dev_err(chan2dev(&atchan->chan_common), "not enough descriptors available\n"); @@ -150,8 +151,9 @@ static void atc_desc_put(struct at_dma_chan *atchan, struct at_desc *desc) { if (desc) { struct at_desc *child; + unsigned long flags; - spin_lock_bh(&atchan->lock); + spin_lock_irqsave(&atchan->lock, flags); list_for_each_entry(child, &desc->tx_list, desc_node) dev_vdbg(chan2dev(&atchan->chan_common), "moving child desc %p to freelist\n", @@ -160,7 +162,7 @@ static void atc_desc_put(struct at_dma_chan *atchan, struct at_desc *desc) dev_vdbg(chan2dev(&atchan->chan_common), "moving desc %p to freelist\n", desc); list_add(&desc->desc_node, &atchan->free_list); - spin_unlock_bh(&atchan->lock); + spin_unlock_irqrestore(&atchan->lock, flags); } } @@ -299,7 +301,7 @@ atc_chain_complete(struct at_dma_chan *atchan, struct at_desc *desc) /* for cyclic transfers, * no need to replay callback function while stopping */ - if (!test_bit(ATC_IS_CYCLIC, &atchan->status)) { + if (!atc_chan_is_cyclic(atchan)) { dma_async_tx_callback callback = txd->callback; void *param = txd->callback_param; @@ -471,16 +473,17 @@ static void atc_handle_cyclic(struct at_dma_chan *atchan) static void atc_tasklet(unsigned long data) { struct at_dma_chan *atchan = (struct at_dma_chan *)data; + unsigned long flags; - spin_lock(&atchan->lock); + spin_lock_irqsave(&atchan->lock, flags); if (test_and_clear_bit(ATC_IS_ERROR, &atchan->status)) atc_handle_error(atchan); - else if (test_bit(ATC_IS_CYCLIC, &atchan->status)) + else if (atc_chan_is_cyclic(atchan)) atc_handle_cyclic(atchan); else atc_advance_work(atchan); - spin_unlock(&atchan->lock); + spin_unlock_irqrestore(&atchan->lock, flags); } static irqreturn_t at_dma_interrupt(int irq, void *dev_id) @@ -539,8 +542,9 @@ static dma_cookie_t atc_tx_submit(struct dma_async_tx_descriptor *tx) struct at_desc *desc = txd_to_at_desc(tx); struct at_dma_chan *atchan = to_at_dma_chan(tx->chan); dma_cookie_t cookie; + unsigned long flags; - spin_lock_bh(&atchan->lock); + spin_lock_irqsave(&atchan->lock, flags); cookie = atc_assign_cookie(atchan, desc); if (list_empty(&atchan->active_list)) { @@ -554,7 +558,7 @@ static dma_cookie_t atc_tx_submit(struct dma_async_tx_descriptor *tx) list_add_tail(&desc->desc_node, &atchan->queue); } - spin_unlock_bh(&atchan->lock); + spin_unlock_irqrestore(&atchan->lock, flags); return cookie; } @@ -927,28 +931,29 @@ static int atc_control(struct dma_chan *chan, enum dma_ctrl_cmd cmd, struct at_dma_chan *atchan = to_at_dma_chan(chan); struct at_dma *atdma = to_at_dma(chan->device); int chan_id = atchan->chan_common.chan_id; + unsigned long flags; LIST_HEAD(list); dev_vdbg(chan2dev(chan), "atc_control (%d)\n", cmd); if (cmd == DMA_PAUSE) { - spin_lock_bh(&atchan->lock); + spin_lock_irqsave(&atchan->lock, flags); dma_writel(atdma, CHER, AT_DMA_SUSP(chan_id)); set_bit(ATC_IS_PAUSED, &atchan->status); - spin_unlock_bh(&atchan->lock); + spin_unlock_irqrestore(&atchan->lock, flags); } else if (cmd == DMA_RESUME) { - if (!test_bit(ATC_IS_PAUSED, &atchan->status)) + if (!atc_chan_is_paused(atchan)) return 0; - spin_lock_bh(&atchan->lock); + spin_lock_irqsave(&atchan->lock, flags); dma_writel(atdma, CHDR, AT_DMA_RES(chan_id)); clear_bit(ATC_IS_PAUSED, &atchan->status); - spin_unlock_bh(&atchan->lock); + spin_unlock_irqrestore(&atchan->lock, flags); } else if (cmd == DMA_TERMINATE_ALL) { struct at_desc *desc, *_desc; /* @@ -957,7 +962,7 @@ static int atc_control(struct dma_chan *chan, enum dma_ctrl_cmd cmd, * channel. We still have to poll the channel enable bit due * to AHB/HSB limitations. */ - spin_lock_bh(&atchan->lock); + spin_lock_irqsave(&atchan->lock, flags); /* disabling channel: must also remove suspend state */ dma_writel(atdma, CHDR, AT_DMA_RES(chan_id) | atchan->mask); @@ -978,7 +983,7 @@ static int atc_control(struct dma_chan *chan, enum dma_ctrl_cmd cmd, /* if channel dedicated to cyclic operations, free it */ clear_bit(ATC_IS_CYCLIC, &atchan->status); - spin_unlock_bh(&atchan->lock); + spin_unlock_irqrestore(&atchan->lock, flags); } else { return -ENXIO; } @@ -1004,9 +1009,10 @@ atc_tx_status(struct dma_chan *chan, struct at_dma_chan *atchan = to_at_dma_chan(chan); dma_cookie_t last_used; dma_cookie_t last_complete; + unsigned long flags; enum dma_status ret; - spin_lock_bh(&atchan->lock); + spin_lock_irqsave(&atchan->lock, flags); last_complete = atchan->completed_cookie; last_used = chan->cookie; @@ -1021,7 +1027,7 @@ atc_tx_status(struct dma_chan *chan, ret = dma_async_is_complete(cookie, last_complete, last_used); } - spin_unlock_bh(&atchan->lock); + spin_unlock_irqrestore(&atchan->lock, flags); if (ret != DMA_SUCCESS) dma_set_tx_state(txstate, last_complete, last_used, @@ -1029,7 +1035,7 @@ atc_tx_status(struct dma_chan *chan, else dma_set_tx_state(txstate, last_complete, last_used, 0); - if (test_bit(ATC_IS_PAUSED, &atchan->status)) + if (atc_chan_is_paused(atchan)) ret = DMA_PAUSED; dev_vdbg(chan2dev(chan), "tx_status %d: cookie = %d (d%d, u%d)\n", @@ -1046,18 +1052,19 @@ atc_tx_status(struct dma_chan *chan, static void atc_issue_pending(struct dma_chan *chan) { struct at_dma_chan *atchan = to_at_dma_chan(chan); + unsigned long flags; dev_vdbg(chan2dev(chan), "issue_pending\n"); /* Not needed for cyclic transfers */ - if (test_bit(ATC_IS_CYCLIC, &atchan->status)) + if (atc_chan_is_cyclic(atchan)) return; - spin_lock_bh(&atchan->lock); + spin_lock_irqsave(&atchan->lock, flags); if (!atc_chan_is_enabled(atchan)) { atc_advance_work(atchan); } - spin_unlock_bh(&atchan->lock); + spin_unlock_irqrestore(&atchan->lock, flags); } /** @@ -1073,6 +1080,7 @@ static int atc_alloc_chan_resources(struct dma_chan *chan) struct at_dma *atdma = to_at_dma(chan->device); struct at_desc *desc; struct at_dma_slave *atslave; + unsigned long flags; int i; u32 cfg; LIST_HEAD(tmp_list); @@ -1116,11 +1124,11 @@ static int atc_alloc_chan_resources(struct dma_chan *chan) list_add_tail(&desc->desc_node, &tmp_list); } - spin_lock_bh(&atchan->lock); + spin_lock_irqsave(&atchan->lock, flags); atchan->descs_allocated = i; list_splice(&tmp_list, &atchan->free_list); atchan->completed_cookie = chan->cookie = 1; - spin_unlock_bh(&atchan->lock); + spin_unlock_irqrestore(&atchan->lock, flags); /* channel parameters */ channel_writel(atchan, CFG, cfg); @@ -1260,12 +1268,11 @@ static int __init at_dma_probe(struct platform_device *pdev) /* initialize channels related values */ INIT_LIST_HEAD(&atdma->dma_common.channels); - for (i = 0; i < pdata->nr_channels; i++, atdma->dma_common.chancnt++) { + for (i = 0; i < pdata->nr_channels; i++) { struct at_dma_chan *atchan = &atdma->chan[i]; atchan->chan_common.device = &atdma->dma_common; atchan->chan_common.cookie = atchan->completed_cookie = 1; - atchan->chan_common.chan_id = i; list_add_tail(&atchan->chan_common.device_node, &atdma->dma_common.channels); @@ -1293,22 +1300,20 @@ static int __init at_dma_probe(struct platform_device *pdev) if (dma_has_cap(DMA_MEMCPY, atdma->dma_common.cap_mask)) atdma->dma_common.device_prep_dma_memcpy = atc_prep_dma_memcpy; - if (dma_has_cap(DMA_SLAVE, atdma->dma_common.cap_mask)) + if (dma_has_cap(DMA_SLAVE, atdma->dma_common.cap_mask)) { atdma->dma_common.device_prep_slave_sg = atc_prep_slave_sg; - - if (dma_has_cap(DMA_CYCLIC, atdma->dma_common.cap_mask)) + /* controller can do slave DMA: can trigger cyclic transfers */ + dma_cap_set(DMA_CYCLIC, atdma->dma_common.cap_mask); atdma->dma_common.device_prep_dma_cyclic = atc_prep_dma_cyclic; - - if (dma_has_cap(DMA_SLAVE, atdma->dma_common.cap_mask) || - dma_has_cap(DMA_CYCLIC, atdma->dma_common.cap_mask)) atdma->dma_common.device_control = atc_control; + } dma_writel(atdma, EN, AT_DMA_ENABLE); dev_info(&pdev->dev, "Atmel AHB DMA Controller ( %s%s), %d channels\n", dma_has_cap(DMA_MEMCPY, atdma->dma_common.cap_mask) ? "cpy " : "", dma_has_cap(DMA_SLAVE, atdma->dma_common.cap_mask) ? "slave " : "", - atdma->dma_common.chancnt); + pdata->nr_channels); dma_async_device_register(&atdma->dma_common); @@ -1377,27 +1382,112 @@ static void at_dma_shutdown(struct platform_device *pdev) clk_disable(atdma->clk); } +static int at_dma_prepare(struct device *dev) +{ + struct platform_device *pdev = to_platform_device(dev); + struct at_dma *atdma = platform_get_drvdata(pdev); + struct dma_chan *chan, *_chan; + + list_for_each_entry_safe(chan, _chan, &atdma->dma_common.channels, + device_node) { + struct at_dma_chan *atchan = to_at_dma_chan(chan); + /* wait for transaction completion (except in cyclic case) */ + if (atc_chan_is_enabled(atchan) && !atc_chan_is_cyclic(atchan)) + return -EAGAIN; + } + return 0; +} + +static void atc_suspend_cyclic(struct at_dma_chan *atchan) +{ + struct dma_chan *chan = &atchan->chan_common; + + /* Channel should be paused by user + * do it anyway even if it is not done already */ + if (!atc_chan_is_paused(atchan)) { + dev_warn(chan2dev(chan), + "cyclic channel not paused, should be done by channel user\n"); + atc_control(chan, DMA_PAUSE, 0); + } + + /* now preserve additional data for cyclic operations */ + /* next descriptor address in the cyclic list */ + atchan->save_dscr = channel_readl(atchan, DSCR); + + vdbg_dump_regs(atchan); +} + static int at_dma_suspend_noirq(struct device *dev) { struct platform_device *pdev = to_platform_device(dev); struct at_dma *atdma = platform_get_drvdata(pdev); + struct dma_chan *chan, *_chan; - at_dma_off(platform_get_drvdata(pdev)); + /* preserve data */ + list_for_each_entry_safe(chan, _chan, &atdma->dma_common.channels, + device_node) { + struct at_dma_chan *atchan = to_at_dma_chan(chan); + + if (atc_chan_is_cyclic(atchan)) + atc_suspend_cyclic(atchan); + atchan->save_cfg = channel_readl(atchan, CFG); + } + atdma->save_imr = dma_readl(atdma, EBCIMR); + + /* disable DMA controller */ + at_dma_off(atdma); clk_disable(atdma->clk); return 0; } +static void atc_resume_cyclic(struct at_dma_chan *atchan) +{ + struct at_dma *atdma = to_at_dma(atchan->chan_common.device); + + /* restore channel status for cyclic descriptors list: + * next descriptor in the cyclic list at the time of suspend */ + channel_writel(atchan, SADDR, 0); + channel_writel(atchan, DADDR, 0); + channel_writel(atchan, CTRLA, 0); + channel_writel(atchan, CTRLB, 0); + channel_writel(atchan, DSCR, atchan->save_dscr); + dma_writel(atdma, CHER, atchan->mask); + + /* channel pause status should be removed by channel user + * We cannot take the initiative to do it here */ + + vdbg_dump_regs(atchan); +} + static int at_dma_resume_noirq(struct device *dev) { struct platform_device *pdev = to_platform_device(dev); struct at_dma *atdma = platform_get_drvdata(pdev); + struct dma_chan *chan, *_chan; + /* bring back DMA controller */ clk_enable(atdma->clk); dma_writel(atdma, EN, AT_DMA_ENABLE); + + /* clear any pending interrupt */ + while (dma_readl(atdma, EBCISR)) + cpu_relax(); + + /* restore saved data */ + dma_writel(atdma, EBCIER, atdma->save_imr); + list_for_each_entry_safe(chan, _chan, &atdma->dma_common.channels, + device_node) { + struct at_dma_chan *atchan = to_at_dma_chan(chan); + + channel_writel(atchan, CFG, atchan->save_cfg); + if (atc_chan_is_cyclic(atchan)) + atc_resume_cyclic(atchan); + } return 0; } static const struct dev_pm_ops at_dma_dev_pm_ops = { + .prepare = at_dma_prepare, .suspend_noirq = at_dma_suspend_noirq, .resume_noirq = at_dma_resume_noirq, }; diff --git a/drivers/dma/at_hdmac_regs.h b/drivers/dma/at_hdmac_regs.h index 087dbf1dd39..aa4c9aebab7 100644 --- a/drivers/dma/at_hdmac_regs.h +++ b/drivers/dma/at_hdmac_regs.h @@ -204,6 +204,9 @@ enum atc_status { * @status: transmit status information from irq/prep* functions * to tasklet (use atomic operations) * @tasklet: bottom half to finish transaction work + * @save_cfg: configuration register that is saved on suspend/resume cycle + * @save_dscr: for cyclic operations, preserve next descriptor address in + * the cyclic list on suspend/resume cycle * @lock: serializes enqueue/dequeue operations to descriptors lists * @completed_cookie: identifier for the most recently completed operation * @active_list: list of descriptors dmaengine is being running on @@ -218,6 +221,8 @@ struct at_dma_chan { u8 mask; unsigned long status; struct tasklet_struct tasklet; + u32 save_cfg; + u32 save_dscr; spinlock_t lock; @@ -248,6 +253,7 @@ static inline struct at_dma_chan *to_at_dma_chan(struct dma_chan *dchan) * @chan_common: common dmaengine dma_device object members * @ch_regs: memory mapped register base * @clk: dma controller clock + * @save_imr: interrupt mask register that is saved on suspend/resume cycle * @all_chan_mask: all channels availlable in a mask * @dma_desc_pool: base of DMA descriptor region (DMA address) * @chan: channels table to store at_dma_chan structures @@ -256,6 +262,7 @@ struct at_dma { struct dma_device dma_common; void __iomem *regs; struct clk *clk; + u32 save_imr; u8 all_chan_mask; @@ -355,6 +362,23 @@ static inline int atc_chan_is_enabled(struct at_dma_chan *atchan) return !!(dma_readl(atdma, CHSR) & atchan->mask); } +/** + * atc_chan_is_paused - test channel pause/resume status + * @atchan: channel we want to test status + */ +static inline int atc_chan_is_paused(struct at_dma_chan *atchan) +{ + return test_bit(ATC_IS_PAUSED, &atchan->status); +} + +/** + * atc_chan_is_cyclic - test if given channel has cyclic property set + * @atchan: channel we want to test status + */ +static inline int atc_chan_is_cyclic(struct at_dma_chan *atchan) +{ + return test_bit(ATC_IS_CYCLIC, &atchan->status); +} /** * set_desc_eol - set end-of-link to descriptor so it will end transfer diff --git a/drivers/dma/dmatest.c b/drivers/dma/dmatest.c index 765f5ff2230..eb1d8641cf5 100644 --- a/drivers/dma/dmatest.c +++ b/drivers/dma/dmatest.c @@ -10,6 +10,7 @@ #include <linux/delay.h> #include <linux/dma-mapping.h> #include <linux/dmaengine.h> +#include <linux/freezer.h> #include <linux/init.h> #include <linux/kthread.h> #include <linux/module.h> @@ -251,6 +252,7 @@ static int dmatest_func(void *data) int i; thread_name = current->comm; + set_freezable_with_signal(); ret = -ENOMEM; @@ -305,7 +307,8 @@ static int dmatest_func(void *data) dma_addr_t dma_srcs[src_cnt]; dma_addr_t dma_dsts[dst_cnt]; struct completion cmp; - unsigned long tmo = msecs_to_jiffies(timeout); + unsigned long start, tmo, end = 0 /* compiler... */; + bool reload = true; u8 align = 0; total_tests++; @@ -404,7 +407,17 @@ static int dmatest_func(void *data) } dma_async_issue_pending(chan); - tmo = wait_for_completion_timeout(&cmp, tmo); + do { + start = jiffies; + if (reload) + end = start + msecs_to_jiffies(timeout); + else if (end <= start) + end = start + 1; + tmo = wait_for_completion_interruptible_timeout(&cmp, + end - start); + reload = try_to_freeze(); + } while (tmo == -ERESTARTSYS); + status = dma_async_is_tx_complete(chan, cookie, NULL, NULL); if (tmo == 0) { @@ -477,6 +490,8 @@ err_srcs: pr_notice("%s: terminating after %u tests, %u failures (status %d)\n", thread_name, total_tests, failed_tests, ret); + /* terminate all transfers on specified channels */ + chan->device->device_control(chan, DMA_TERMINATE_ALL, 0); if (iterations > 0) while (!kthread_should_stop()) { DECLARE_WAIT_QUEUE_HEAD_ONSTACK(wait_dmatest_exit); @@ -499,6 +514,10 @@ static void dmatest_cleanup_channel(struct dmatest_chan *dtc) list_del(&thread->node); kfree(thread); } + + /* terminate all transfers on specified channels */ + dtc->chan->device->device_control(dtc->chan, DMA_TERMINATE_ALL, 0); + kfree(dtc); } diff --git a/drivers/dma/dw_dmac.c b/drivers/dma/dw_dmac.c index 4d180ca9a1d..9bfd6d36071 100644 --- a/drivers/dma/dw_dmac.c +++ b/drivers/dma/dw_dmac.c @@ -1407,12 +1407,11 @@ static int __init dw_probe(struct platform_device *pdev) dw->all_chan_mask = (1 << pdata->nr_channels) - 1; INIT_LIST_HEAD(&dw->dma.channels); - for (i = 0; i < pdata->nr_channels; i++, dw->dma.chancnt++) { + for (i = 0; i < pdata->nr_channels; i++) { struct dw_dma_chan *dwc = &dw->chan[i]; dwc->chan.device = &dw->dma; dwc->chan.cookie = dwc->completed = 1; - dwc->chan.chan_id = i; if (pdata->chan_allocation_order == CHAN_ALLOCATION_ASCENDING) list_add_tail(&dwc->chan.device_node, &dw->dma.channels); @@ -1468,7 +1467,7 @@ static int __init dw_probe(struct platform_device *pdev) dma_writel(dw, CFG, DW_CFG_DMA_EN); printk(KERN_INFO "%s: DesignWare DMA Controller, %d channels\n", - dev_name(&pdev->dev), dw->dma.chancnt); + dev_name(&pdev->dev), pdata->nr_channels); dma_async_device_register(&dw->dma); diff --git a/drivers/dma/ep93xx_dma.c b/drivers/dma/ep93xx_dma.c index 5d7a49bd7c2..b47e2b803fa 100644 --- a/drivers/dma/ep93xx_dma.c +++ b/drivers/dma/ep93xx_dma.c @@ -22,6 +22,7 @@ #include <linux/init.h> #include <linux/interrupt.h> #include <linux/dmaengine.h> +#include <linux/module.h> #include <linux/platform_device.h> #include <linux/slab.h> diff --git a/drivers/dma/imx-dma.c b/drivers/dma/imx-dma.c index d99f71c356b..4be55f9bb6c 100644 --- a/drivers/dma/imx-dma.c +++ b/drivers/dma/imx-dma.c @@ -14,6 +14,7 @@ * http://www.gnu.org/copyleft/gpl.html */ #include <linux/init.h> +#include <linux/module.h> #include <linux/types.h> #include <linux/mm.h> #include <linux/interrupt.h> @@ -23,6 +24,7 @@ #include <linux/slab.h> #include <linux/platform_device.h> #include <linux/dmaengine.h> +#include <linux/module.h> #include <asm/irq.h> #include <mach/dma-v1.h> diff --git a/drivers/dma/imx-sdma.c b/drivers/dma/imx-sdma.c index 7bd7e98548c..f993955a640 100644 --- a/drivers/dma/imx-sdma.c +++ b/drivers/dma/imx-sdma.c @@ -18,6 +18,7 @@ */ #include <linux/init.h> +#include <linux/module.h> #include <linux/types.h> #include <linux/mm.h> #include <linux/interrupt.h> @@ -34,6 +35,7 @@ #include <linux/dmaengine.h> #include <linux/of.h> #include <linux/of_device.h> +#include <linux/module.h> #include <asm/irq.h> #include <mach/sdma.h> @@ -318,6 +320,7 @@ struct sdma_engine { dma_addr_t context_phys; struct dma_device dma_device; struct clk *clk; + struct mutex channel_0_lock; struct sdma_script_start_addrs *script_addrs; }; @@ -415,11 +418,15 @@ static int sdma_load_script(struct sdma_engine *sdma, void *buf, int size, dma_addr_t buf_phys; int ret; + mutex_lock(&sdma->channel_0_lock); + buf_virt = dma_alloc_coherent(NULL, size, &buf_phys, GFP_KERNEL); - if (!buf_virt) - return -ENOMEM; + if (!buf_virt) { + ret = -ENOMEM; + goto err_out; + } bd0->mode.command = C0_SETPM; bd0->mode.status = BD_DONE | BD_INTR | BD_WRAP | BD_EXTD; @@ -433,6 +440,9 @@ static int sdma_load_script(struct sdma_engine *sdma, void *buf, int size, dma_free_coherent(NULL, size, buf_virt, buf_phys); +err_out: + mutex_unlock(&sdma->channel_0_lock); + return ret; } @@ -656,6 +666,8 @@ static int sdma_load_context(struct sdma_channel *sdmac) dev_dbg(sdma->dev, "event_mask0 = 0x%08x\n", sdmac->event_mask0); dev_dbg(sdma->dev, "event_mask1 = 0x%08x\n", sdmac->event_mask1); + mutex_lock(&sdma->channel_0_lock); + memset(context, 0, sizeof(*context)); context->channel_state.pc = load_address; @@ -676,6 +688,8 @@ static int sdma_load_context(struct sdma_channel *sdmac) ret = sdma_run_channel(&sdma->channel[0]); + mutex_unlock(&sdma->channel_0_lock); + return ret; } @@ -1131,18 +1145,17 @@ static void sdma_add_scripts(struct sdma_engine *sdma, saddr_arr[i] = addr_arr[i]; } -static int __init sdma_get_firmware(struct sdma_engine *sdma, - const char *fw_name) +static void sdma_load_firmware(const struct firmware *fw, void *context) { - const struct firmware *fw; + struct sdma_engine *sdma = context; const struct sdma_firmware_header *header; - int ret; const struct sdma_script_start_addrs *addr; unsigned short *ram_code; - ret = request_firmware(&fw, fw_name, sdma->dev); - if (ret) - return ret; + if (!fw) { + dev_err(sdma->dev, "firmware not found\n"); + return; + } if (fw->size < sizeof(*header)) goto err_firmware; @@ -1172,6 +1185,16 @@ static int __init sdma_get_firmware(struct sdma_engine *sdma, err_firmware: release_firmware(fw); +} + +static int __init sdma_get_firmware(struct sdma_engine *sdma, + const char *fw_name) +{ + int ret; + + ret = request_firmware_nowait(THIS_MODULE, + FW_ACTION_HOTPLUG, fw_name, sdma->dev, + GFP_KERNEL, sdma, sdma_load_firmware); return ret; } @@ -1269,11 +1292,14 @@ static int __init sdma_probe(struct platform_device *pdev) struct sdma_platform_data *pdata = pdev->dev.platform_data; int i; struct sdma_engine *sdma; + s32 *saddr_arr; sdma = kzalloc(sizeof(*sdma), GFP_KERNEL); if (!sdma) return -ENOMEM; + mutex_init(&sdma->channel_0_lock); + sdma->dev = &pdev->dev; iores = platform_get_resource(pdev, IORESOURCE_MEM, 0); @@ -1310,6 +1336,11 @@ static int __init sdma_probe(struct platform_device *pdev) goto err_alloc; } + /* initially no scripts available */ + saddr_arr = (s32 *)sdma->script_addrs; + for (i = 0; i < SDMA_SCRIPT_ADDRS_ARRAY_SIZE_V1; i++) + saddr_arr[i] = -EINVAL; + if (of_id) pdev->id_entry = of_id->data; sdma->devtype = pdev->id_entry->driver_data; diff --git a/drivers/dma/intel_mid_dma.c b/drivers/dma/intel_mid_dma.c index 8a3fdd87db9..19a0c64d45d 100644 --- a/drivers/dma/intel_mid_dma.c +++ b/drivers/dma/intel_mid_dma.c @@ -27,6 +27,7 @@ #include <linux/interrupt.h> #include <linux/pm_runtime.h> #include <linux/intel_mid_dma.h> +#include <linux/module.h> #define MAX_CHAN 4 /*max ch across controllers*/ #include "intel_mid_dma_regs.h" @@ -115,16 +116,15 @@ DMAC1 interrupt Functions*/ /** * dmac1_mask_periphral_intr - mask the periphral interrupt - * @midc: dma channel for which masking is required + * @mid: dma device for which masking is required * * Masks the DMA periphral interrupt * this is valid for DMAC1 family controllers only * This controller should have periphral mask registers already mapped */ -static void dmac1_mask_periphral_intr(struct intel_mid_dma_chan *midc) +static void dmac1_mask_periphral_intr(struct middma_device *mid) { u32 pimr; - struct middma_device *mid = to_middma_device(midc->chan.device); if (mid->pimr_mask) { pimr = readl(mid->mask_reg + LNW_PERIPHRAL_MASK); @@ -184,7 +184,6 @@ static void enable_dma_interrupt(struct intel_mid_dma_chan *midc) static void disable_dma_interrupt(struct intel_mid_dma_chan *midc) { /*Check LPE PISR, make sure fwd is disabled*/ - dmac1_mask_periphral_intr(midc); iowrite32(MASK_INTR_REG(midc->ch_id), midc->dma_base + MASK_BLOCK); iowrite32(MASK_INTR_REG(midc->ch_id), midc->dma_base + MASK_TFR); iowrite32(MASK_INTR_REG(midc->ch_id), midc->dma_base + MASK_ERR); @@ -1114,7 +1113,6 @@ static int mid_setup_dma(struct pci_dev *pdev) midch->chan.device = &dma->common; midch->chan.cookie = 1; - midch->chan.chan_id = i; midch->ch_id = dma->chan_base + i; pr_debug("MDMA:Init CH %d, ID %d\n", i, midch->ch_id); @@ -1150,7 +1148,6 @@ static int mid_setup_dma(struct pci_dev *pdev) dma_cap_set(DMA_SLAVE, dma->common.cap_mask); dma_cap_set(DMA_PRIVATE, dma->common.cap_mask); dma->common.dev = &pdev->dev; - dma->common.chancnt = dma->max_chan; dma->common.device_alloc_chan_resources = intel_mid_dma_alloc_chan_resources; @@ -1350,6 +1347,7 @@ int dma_suspend(struct pci_dev *pci, pm_message_t state) if (device->ch[i].in_use) return -EAGAIN; } + dmac1_mask_periphral_intr(device); device->state = SUSPENDED; pci_save_state(pci); pci_disable_device(pci); diff --git a/drivers/dma/ipu/ipu_idmac.c b/drivers/dma/ipu/ipu_idmac.c index 6815905a772..0e5ef33f90a 100644 --- a/drivers/dma/ipu/ipu_idmac.c +++ b/drivers/dma/ipu/ipu_idmac.c @@ -21,6 +21,7 @@ #include <linux/string.h> #include <linux/interrupt.h> #include <linux/io.h> +#include <linux/module.h> #include <mach/ipu.h> @@ -1307,6 +1308,7 @@ static irqreturn_t idmac_interrupt(int irq, void *dev_id) ipu_submit_buffer(ichan, descnew, sgnew, ichan->active_buffer) < 0) { callback = descnew->txd.callback; callback_param = descnew->txd.callback_param; + list_del_init(&descnew->list); spin_unlock(&ichan->lock); if (callback) callback(callback_param); @@ -1428,39 +1430,58 @@ static int __idmac_control(struct dma_chan *chan, enum dma_ctrl_cmd cmd, { struct idmac_channel *ichan = to_idmac_chan(chan); struct idmac *idmac = to_idmac(chan->device); + struct ipu *ipu = to_ipu(idmac); + struct list_head *list, *tmp; unsigned long flags; int i; - /* Only supports DMA_TERMINATE_ALL */ - if (cmd != DMA_TERMINATE_ALL) - return -ENXIO; + switch (cmd) { + case DMA_PAUSE: + spin_lock_irqsave(&ipu->lock, flags); + ipu_ic_disable_task(ipu, chan->chan_id); - ipu_disable_channel(idmac, ichan, - ichan->status >= IPU_CHANNEL_ENABLED); + /* Return all descriptors into "prepared" state */ + list_for_each_safe(list, tmp, &ichan->queue) + list_del_init(list); - tasklet_disable(&to_ipu(idmac)->tasklet); + ichan->sg[0] = NULL; + ichan->sg[1] = NULL; - /* ichan->queue is modified in ISR, have to spinlock */ - spin_lock_irqsave(&ichan->lock, flags); - list_splice_init(&ichan->queue, &ichan->free_list); + spin_unlock_irqrestore(&ipu->lock, flags); - if (ichan->desc) - for (i = 0; i < ichan->n_tx_desc; i++) { - struct idmac_tx_desc *desc = ichan->desc + i; - if (list_empty(&desc->list)) - /* Descriptor was prepared, but not submitted */ - list_add(&desc->list, &ichan->free_list); + ichan->status = IPU_CHANNEL_INITIALIZED; + break; + case DMA_TERMINATE_ALL: + ipu_disable_channel(idmac, ichan, + ichan->status >= IPU_CHANNEL_ENABLED); - async_tx_clear_ack(&desc->txd); - } + tasklet_disable(&ipu->tasklet); - ichan->sg[0] = NULL; - ichan->sg[1] = NULL; - spin_unlock_irqrestore(&ichan->lock, flags); + /* ichan->queue is modified in ISR, have to spinlock */ + spin_lock_irqsave(&ichan->lock, flags); + list_splice_init(&ichan->queue, &ichan->free_list); - tasklet_enable(&to_ipu(idmac)->tasklet); + if (ichan->desc) + for (i = 0; i < ichan->n_tx_desc; i++) { + struct idmac_tx_desc *desc = ichan->desc + i; + if (list_empty(&desc->list)) + /* Descriptor was prepared, but not submitted */ + list_add(&desc->list, &ichan->free_list); - ichan->status = IPU_CHANNEL_INITIALIZED; + async_tx_clear_ack(&desc->txd); + } + + ichan->sg[0] = NULL; + ichan->sg[1] = NULL; + spin_unlock_irqrestore(&ichan->lock, flags); + + tasklet_enable(&ipu->tasklet); + + ichan->status = IPU_CHANNEL_INITIALIZED; + break; + default: + return -ENOSYS; + } return 0; } @@ -1663,7 +1684,6 @@ static void __exit ipu_idmac_exit(struct ipu *ipu) struct idmac_channel *ichan = ipu->channel + i; idmac_control(&ichan->dma_chan, DMA_TERMINATE_ALL, 0); - idmac_prep_slave_sg(&ichan->dma_chan, NULL, 0, DMA_NONE, 0); } dma_async_device_unregister(&idmac->dma); diff --git a/drivers/dma/mpc512x_dma.c b/drivers/dma/mpc512x_dma.c index b9bae94f201..8ba4edc6185 100644 --- a/drivers/dma/mpc512x_dma.c +++ b/drivers/dma/mpc512x_dma.c @@ -741,7 +741,6 @@ static int __devinit mpc_dma_probe(struct platform_device *op) mchan = &mdma->channels[i]; mchan->chan.device = dma; - mchan->chan.chan_id = i; mchan->chan.cookie = 1; mchan->completed_cookie = mchan->chan.cookie; diff --git a/drivers/dma/mxs-dma.c b/drivers/dma/mxs-dma.c index be641cbd36f..b4588bdd98b 100644 --- a/drivers/dma/mxs-dma.c +++ b/drivers/dma/mxs-dma.c @@ -130,6 +130,23 @@ struct mxs_dma_engine { struct mxs_dma_chan mxs_chans[MXS_DMA_CHANNELS]; }; +static inline void mxs_dma_clkgate(struct mxs_dma_chan *mxs_chan, int enable) +{ + struct mxs_dma_engine *mxs_dma = mxs_chan->mxs_dma; + int chan_id = mxs_chan->chan.chan_id; + int set_clr = enable ? MXS_CLR_ADDR : MXS_SET_ADDR; + + /* enable apbh channel clock */ + if (dma_is_apbh()) { + if (apbh_is_old()) + writel(1 << (chan_id + BP_APBH_CTRL0_CLKGATE_CHANNEL), + mxs_dma->base + HW_APBHX_CTRL0 + set_clr); + else + writel(1 << chan_id, + mxs_dma->base + HW_APBHX_CTRL0 + set_clr); + } +} + static void mxs_dma_reset_chan(struct mxs_dma_chan *mxs_chan) { struct mxs_dma_engine *mxs_dma = mxs_chan->mxs_dma; @@ -148,38 +165,21 @@ static void mxs_dma_enable_chan(struct mxs_dma_chan *mxs_chan) struct mxs_dma_engine *mxs_dma = mxs_chan->mxs_dma; int chan_id = mxs_chan->chan.chan_id; + /* clkgate needs to be enabled before writing other registers */ + mxs_dma_clkgate(mxs_chan, 1); + /* set cmd_addr up */ writel(mxs_chan->ccw_phys, mxs_dma->base + HW_APBHX_CHn_NXTCMDAR(chan_id)); - /* enable apbh channel clock */ - if (dma_is_apbh()) { - if (apbh_is_old()) - writel(1 << (chan_id + BP_APBH_CTRL0_CLKGATE_CHANNEL), - mxs_dma->base + HW_APBHX_CTRL0 + MXS_CLR_ADDR); - else - writel(1 << chan_id, - mxs_dma->base + HW_APBHX_CTRL0 + MXS_CLR_ADDR); - } - /* write 1 to SEMA to kick off the channel */ writel(1, mxs_dma->base + HW_APBHX_CHn_SEMA(chan_id)); } static void mxs_dma_disable_chan(struct mxs_dma_chan *mxs_chan) { - struct mxs_dma_engine *mxs_dma = mxs_chan->mxs_dma; - int chan_id = mxs_chan->chan.chan_id; - /* disable apbh channel clock */ - if (dma_is_apbh()) { - if (apbh_is_old()) - writel(1 << (chan_id + BP_APBH_CTRL0_CLKGATE_CHANNEL), - mxs_dma->base + HW_APBHX_CTRL0 + MXS_SET_ADDR); - else - writel(1 << chan_id, - mxs_dma->base + HW_APBHX_CTRL0 + MXS_SET_ADDR); - } + mxs_dma_clkgate(mxs_chan, 0); mxs_chan->status = DMA_SUCCESS; } @@ -338,7 +338,10 @@ static int mxs_dma_alloc_chan_resources(struct dma_chan *chan) if (ret) goto err_clk; + /* clkgate needs to be enabled for reset to finish */ + mxs_dma_clkgate(mxs_chan, 1); mxs_dma_reset_chan(mxs_chan); + mxs_dma_clkgate(mxs_chan, 0); dma_async_tx_descriptor_init(&mxs_chan->desc, chan); mxs_chan->desc.tx_submit = mxs_dma_tx_submit; diff --git a/drivers/dma/pch_dma.c b/drivers/dma/pch_dma.c index 1ac8d4b580b..a6d0e3dbed0 100644 --- a/drivers/dma/pch_dma.c +++ b/drivers/dma/pch_dma.c @@ -60,7 +60,7 @@ #define DMA_DESC_FOLLOW_WITHOUT_IRQ 0x2 #define DMA_DESC_FOLLOW_WITH_IRQ 0x3 -#define MAX_CHAN_NR 8 +#define MAX_CHAN_NR 12 #define DMA_MASK_CTL0_MODE 0x33333333 #define DMA_MASK_CTL2_MODE 0x00003333 @@ -872,8 +872,7 @@ static int __devinit pch_dma_probe(struct pci_dev *pdev, int i; nr_channels = id->driver_data; - pd = kzalloc(sizeof(struct pch_dma)+ - sizeof(struct pch_dma_chan) * nr_channels, GFP_KERNEL); + pd = kzalloc(sizeof(*pd), GFP_KERNEL); if (!pd) return -ENOMEM; @@ -926,7 +925,6 @@ static int __devinit pch_dma_probe(struct pci_dev *pdev, } pd->dma.dev = &pdev->dev; - pd->dma.chancnt = nr_channels; INIT_LIST_HEAD(&pd->dma.channels); @@ -935,7 +933,6 @@ static int __devinit pch_dma_probe(struct pci_dev *pdev, pd_chan->chan.device = &pd->dma; pd_chan->chan.cookie = 1; - pd_chan->chan.chan_id = i; pd_chan->membase = ®s->desc[i]; diff --git a/drivers/dma/pl330.c b/drivers/dma/pl330.c index 00eee59e8b3..571041477ab 100644 --- a/drivers/dma/pl330.c +++ b/drivers/dma/pl330.c @@ -17,6 +17,8 @@ #include <linux/interrupt.h> #include <linux/amba/bus.h> #include <linux/amba/pl330.h> +#include <linux/pm_runtime.h> +#include <linux/scatterlist.h> #define NR_DEFAULT_DESC 16 @@ -68,6 +70,14 @@ struct dma_pl330_chan { * NULL if the channel is available to be acquired. */ void *pl330_chid; + + /* For D-to-M and M-to-D channels */ + int burst_sz; /* the peripheral fifo width */ + int burst_len; /* the number of burst */ + dma_addr_t fifo_addr; + + /* for cyclic capability */ + bool cyclic; }; struct dma_pl330_dmac { @@ -83,6 +93,8 @@ struct dma_pl330_dmac { /* Peripheral channels connected to this DMAC */ struct dma_pl330_chan *peripherals; /* keep at end */ + + struct clk *clk; }; struct dma_pl330_desc { @@ -152,6 +164,31 @@ static inline void free_desc_list(struct list_head *list) spin_unlock_irqrestore(&pdmac->pool_lock, flags); } +static inline void handle_cyclic_desc_list(struct list_head *list) +{ + struct dma_pl330_desc *desc; + struct dma_pl330_chan *pch; + unsigned long flags; + + if (list_empty(list)) + return; + + list_for_each_entry(desc, list, node) { + dma_async_tx_callback callback; + + /* Change status to reload it */ + desc->status = PREP; + pch = desc->pchan; + callback = desc->txd.callback; + if (callback) + callback(desc->txd.callback_param); + } + + spin_lock_irqsave(&pch->lock, flags); + list_splice_tail_init(list, &pch->work_list); + spin_unlock_irqrestore(&pch->lock, flags); +} + static inline void fill_queue(struct dma_pl330_chan *pch) { struct dma_pl330_desc *desc; @@ -205,7 +242,10 @@ static void pl330_tasklet(unsigned long data) spin_unlock_irqrestore(&pch->lock, flags); - free_desc_list(&list); + if (pch->cyclic) + handle_cyclic_desc_list(&list); + else + free_desc_list(&list); } static void dma_pl330_rqcb(void *token, enum pl330_op_err err) @@ -236,6 +276,7 @@ static int pl330_alloc_chan_resources(struct dma_chan *chan) spin_lock_irqsave(&pch->lock, flags); pch->completed = chan->cookie = 1; + pch->cyclic = false; pch->pl330_chid = pl330_request_channel(&pdmac->pif); if (!pch->pl330_chid) { @@ -253,25 +294,52 @@ static int pl330_alloc_chan_resources(struct dma_chan *chan) static int pl330_control(struct dma_chan *chan, enum dma_ctrl_cmd cmd, unsigned long arg) { struct dma_pl330_chan *pch = to_pchan(chan); - struct dma_pl330_desc *desc; + struct dma_pl330_desc *desc, *_dt; unsigned long flags; + struct dma_pl330_dmac *pdmac = pch->dmac; + struct dma_slave_config *slave_config; + LIST_HEAD(list); - /* Only supports DMA_TERMINATE_ALL */ - if (cmd != DMA_TERMINATE_ALL) - return -ENXIO; - - spin_lock_irqsave(&pch->lock, flags); - - /* FLUSH the PL330 Channel thread */ - pl330_chan_ctrl(pch->pl330_chid, PL330_OP_FLUSH); + switch (cmd) { + case DMA_TERMINATE_ALL: + spin_lock_irqsave(&pch->lock, flags); - /* Mark all desc done */ - list_for_each_entry(desc, &pch->work_list, node) - desc->status = DONE; + /* FLUSH the PL330 Channel thread */ + pl330_chan_ctrl(pch->pl330_chid, PL330_OP_FLUSH); - spin_unlock_irqrestore(&pch->lock, flags); + /* Mark all desc done */ + list_for_each_entry_safe(desc, _dt, &pch->work_list , node) { + desc->status = DONE; + pch->completed = desc->txd.cookie; + list_move_tail(&desc->node, &list); + } - pl330_tasklet((unsigned long) pch); + list_splice_tail_init(&list, &pdmac->desc_pool); + spin_unlock_irqrestore(&pch->lock, flags); + break; + case DMA_SLAVE_CONFIG: + slave_config = (struct dma_slave_config *)arg; + + if (slave_config->direction == DMA_TO_DEVICE) { + if (slave_config->dst_addr) + pch->fifo_addr = slave_config->dst_addr; + if (slave_config->dst_addr_width) + pch->burst_sz = __ffs(slave_config->dst_addr_width); + if (slave_config->dst_maxburst) + pch->burst_len = slave_config->dst_maxburst; + } else if (slave_config->direction == DMA_FROM_DEVICE) { + if (slave_config->src_addr) + pch->fifo_addr = slave_config->src_addr; + if (slave_config->src_addr_width) + pch->burst_sz = __ffs(slave_config->src_addr_width); + if (slave_config->src_maxburst) + pch->burst_len = slave_config->src_maxburst; + } + break; + default: + dev_err(pch->dmac->pif.dev, "Not supported command.\n"); + return -ENXIO; + } return 0; } @@ -288,6 +356,9 @@ static void pl330_free_chan_resources(struct dma_chan *chan) pl330_release_channel(pch->pl330_chid); pch->pl330_chid = NULL; + if (pch->cyclic) + list_splice_tail_init(&pch->work_list, &pch->dmac->desc_pool); + spin_unlock_irqrestore(&pch->lock, flags); } @@ -453,7 +524,7 @@ static struct dma_pl330_desc *pl330_get_desc(struct dma_pl330_chan *pch) if (peri) { desc->req.rqtype = peri->rqtype; - desc->req.peri = peri->peri_id; + desc->req.peri = pch->chan.chan_id; } else { desc->req.rqtype = MEMTOMEM; desc->req.peri = 0; @@ -524,6 +595,51 @@ static inline int get_burst_len(struct dma_pl330_desc *desc, size_t len) return burst_len; } +static struct dma_async_tx_descriptor *pl330_prep_dma_cyclic( + struct dma_chan *chan, dma_addr_t dma_addr, size_t len, + size_t period_len, enum dma_data_direction direction) +{ + struct dma_pl330_desc *desc; + struct dma_pl330_chan *pch = to_pchan(chan); + dma_addr_t dst; + dma_addr_t src; + + desc = pl330_get_desc(pch); + if (!desc) { + dev_err(pch->dmac->pif.dev, "%s:%d Unable to fetch desc\n", + __func__, __LINE__); + return NULL; + } + + switch (direction) { + case DMA_TO_DEVICE: + desc->rqcfg.src_inc = 1; + desc->rqcfg.dst_inc = 0; + src = dma_addr; + dst = pch->fifo_addr; + break; + case DMA_FROM_DEVICE: + desc->rqcfg.src_inc = 0; + desc->rqcfg.dst_inc = 1; + src = pch->fifo_addr; + dst = dma_addr; + break; + default: + dev_err(pch->dmac->pif.dev, "%s:%d Invalid dma direction\n", + __func__, __LINE__); + return NULL; + } + + desc->rqcfg.brst_size = pch->burst_sz; + desc->rqcfg.brst_len = 1; + + pch->cyclic = true; + + fill_px(&desc->px, dst, src, period_len); + + return &desc->txd; +} + static struct dma_async_tx_descriptor * pl330_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dst, dma_addr_t src, size_t len, unsigned long flags) @@ -579,7 +695,7 @@ pl330_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl, struct dma_pl330_peri *peri = chan->private; struct scatterlist *sg; unsigned long flags; - int i, burst_size; + int i; dma_addr_t addr; if (unlikely(!pch || !sgl || !sg_len || !peri)) @@ -595,8 +711,7 @@ pl330_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl, return NULL; } - addr = peri->fifo_addr; - burst_size = peri->burst_sz; + addr = pch->fifo_addr; first = NULL; @@ -644,7 +759,7 @@ pl330_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl, sg_dma_address(sg), addr, sg_dma_len(sg)); } - desc->rqcfg.brst_size = burst_size; + desc->rqcfg.brst_size = pch->burst_sz; desc->rqcfg.brst_len = 1; } @@ -696,6 +811,30 @@ pl330_probe(struct amba_device *adev, const struct amba_id *id) goto probe_err1; } + pdmac->clk = clk_get(&adev->dev, "dma"); + if (IS_ERR(pdmac->clk)) { + dev_err(&adev->dev, "Cannot get operation clock.\n"); + ret = -EINVAL; + goto probe_err1; + } + + amba_set_drvdata(adev, pdmac); + +#ifdef CONFIG_PM_RUNTIME + /* to use the runtime PM helper functions */ + pm_runtime_enable(&adev->dev); + + /* enable the power domain */ + if (pm_runtime_get_sync(&adev->dev)) { + dev_err(&adev->dev, "failed to get runtime pm\n"); + ret = -ENODEV; + goto probe_err1; + } +#else + /* enable dma clk */ + clk_enable(pdmac->clk); +#endif + irq = adev->irq[0]; ret = request_irq(irq, pl330_irq_handler, 0, dev_name(&adev->dev), pi); @@ -732,6 +871,7 @@ pl330_probe(struct amba_device *adev, const struct amba_id *id) case MEMTODEV: case DEVTOMEM: dma_cap_set(DMA_SLAVE, pd->cap_mask); + dma_cap_set(DMA_CYCLIC, pd->cap_mask); break; default: dev_err(&adev->dev, "DEVTODEV Not Supported\n"); @@ -747,11 +887,9 @@ pl330_probe(struct amba_device *adev, const struct amba_id *id) spin_lock_init(&pch->lock); pch->pl330_chid = NULL; pch->chan.device = pd; - pch->chan.chan_id = i; pch->dmac = pdmac; /* Add the channel to the DMAC list */ - pd->chancnt++; list_add_tail(&pch->chan.device_node, &pd->channels); } @@ -760,6 +898,7 @@ pl330_probe(struct amba_device *adev, const struct amba_id *id) pd->device_alloc_chan_resources = pl330_alloc_chan_resources; pd->device_free_chan_resources = pl330_free_chan_resources; pd->device_prep_dma_memcpy = pl330_prep_dma_memcpy; + pd->device_prep_dma_cyclic = pl330_prep_dma_cyclic; pd->device_tx_status = pl330_tx_status; pd->device_prep_slave_sg = pl330_prep_slave_sg; pd->device_control = pl330_control; @@ -771,8 +910,6 @@ pl330_probe(struct amba_device *adev, const struct amba_id *id) goto probe_err4; } - amba_set_drvdata(adev, pdmac); - dev_info(&adev->dev, "Loaded driver for PL330 DMAC-%d\n", adev->periphid); dev_info(&adev->dev, @@ -833,6 +970,13 @@ static int __devexit pl330_remove(struct amba_device *adev) res = &adev->res; release_mem_region(res->start, resource_size(res)); +#ifdef CONFIG_PM_RUNTIME + pm_runtime_put(&adev->dev); + pm_runtime_disable(&adev->dev); +#else + clk_disable(pdmac->clk); +#endif + kfree(pdmac); return 0; @@ -846,10 +990,49 @@ static struct amba_id pl330_ids[] = { { 0, 0 }, }; +#ifdef CONFIG_PM_RUNTIME +static int pl330_runtime_suspend(struct device *dev) +{ + struct dma_pl330_dmac *pdmac = dev_get_drvdata(dev); + + if (!pdmac) { + dev_err(dev, "failed to get dmac\n"); + return -ENODEV; + } + + clk_disable(pdmac->clk); + + return 0; +} + +static int pl330_runtime_resume(struct device *dev) +{ + struct dma_pl330_dmac *pdmac = dev_get_drvdata(dev); + + if (!pdmac) { + dev_err(dev, "failed to get dmac\n"); + return -ENODEV; + } + + clk_enable(pdmac->clk); + + return 0; +} +#else +#define pl330_runtime_suspend NULL +#define pl330_runtime_resume NULL +#endif /* CONFIG_PM_RUNTIME */ + +static const struct dev_pm_ops pl330_pm_ops = { + .runtime_suspend = pl330_runtime_suspend, + .runtime_resume = pl330_runtime_resume, +}; + static struct amba_driver pl330_driver = { .drv = { .owner = THIS_MODULE, .name = "dma-pl330", + .pm = &pl330_pm_ops, }, .id_table = pl330_ids, .probe = pl330_probe, diff --git a/drivers/dma/shdma.c b/drivers/dma/shdma.c index 7f49235d14b..81809c2b46a 100644 --- a/drivers/dma/shdma.c +++ b/drivers/dma/shdma.c @@ -259,14 +259,23 @@ static int dmae_set_dmars(struct sh_dmae_chan *sh_chan, u16 val) return 0; } +static void sh_chan_xfer_ld_queue(struct sh_dmae_chan *sh_chan); + static dma_cookie_t sh_dmae_tx_submit(struct dma_async_tx_descriptor *tx) { struct sh_desc *desc = tx_to_sh_desc(tx), *chunk, *last = desc, *c; struct sh_dmae_chan *sh_chan = to_sh_chan(tx->chan); + struct sh_dmae_slave *param = tx->chan->private; dma_async_tx_callback callback = tx->callback; dma_cookie_t cookie; + bool power_up; + + spin_lock_irq(&sh_chan->desc_lock); - spin_lock_bh(&sh_chan->desc_lock); + if (list_empty(&sh_chan->ld_queue)) + power_up = true; + else + power_up = false; cookie = sh_chan->common.cookie; cookie++; @@ -302,7 +311,38 @@ static dma_cookie_t sh_dmae_tx_submit(struct dma_async_tx_descriptor *tx) tx->cookie, &last->async_tx, sh_chan->id, desc->hw.sar, desc->hw.tcr, desc->hw.dar); - spin_unlock_bh(&sh_chan->desc_lock); + if (power_up) { + sh_chan->pm_state = DMAE_PM_BUSY; + + pm_runtime_get(sh_chan->dev); + + spin_unlock_irq(&sh_chan->desc_lock); + + pm_runtime_barrier(sh_chan->dev); + + spin_lock_irq(&sh_chan->desc_lock); + + /* Have we been reset, while waiting? */ + if (sh_chan->pm_state != DMAE_PM_ESTABLISHED) { + dev_dbg(sh_chan->dev, "Bring up channel %d\n", + sh_chan->id); + if (param) { + const struct sh_dmae_slave_config *cfg = + param->config; + + dmae_set_dmars(sh_chan, cfg->mid_rid); + dmae_set_chcr(sh_chan, cfg->chcr); + } else { + dmae_init(sh_chan); + } + + if (sh_chan->pm_state == DMAE_PM_PENDING) + sh_chan_xfer_ld_queue(sh_chan); + sh_chan->pm_state = DMAE_PM_ESTABLISHED; + } + } + + spin_unlock_irq(&sh_chan->desc_lock); return cookie; } @@ -346,8 +386,6 @@ static int sh_dmae_alloc_chan_resources(struct dma_chan *chan) struct sh_dmae_slave *param = chan->private; int ret; - pm_runtime_get_sync(sh_chan->dev); - /* * This relies on the guarantee from dmaengine that alloc_chan_resources * never runs concurrently with itself or free_chan_resources. @@ -367,31 +405,20 @@ static int sh_dmae_alloc_chan_resources(struct dma_chan *chan) } param->config = cfg; - - dmae_set_dmars(sh_chan, cfg->mid_rid); - dmae_set_chcr(sh_chan, cfg->chcr); - } else { - dmae_init(sh_chan); } - spin_lock_bh(&sh_chan->desc_lock); while (sh_chan->descs_allocated < NR_DESCS_PER_CHANNEL) { - spin_unlock_bh(&sh_chan->desc_lock); desc = kzalloc(sizeof(struct sh_desc), GFP_KERNEL); - if (!desc) { - spin_lock_bh(&sh_chan->desc_lock); + if (!desc) break; - } dma_async_tx_descriptor_init(&desc->async_tx, &sh_chan->common); desc->async_tx.tx_submit = sh_dmae_tx_submit; desc->mark = DESC_IDLE; - spin_lock_bh(&sh_chan->desc_lock); list_add(&desc->node, &sh_chan->ld_free); sh_chan->descs_allocated++; } - spin_unlock_bh(&sh_chan->desc_lock); if (!sh_chan->descs_allocated) { ret = -ENOMEM; @@ -405,7 +432,7 @@ edescalloc: clear_bit(param->slave_id, sh_dmae_slave_used); etestused: efindslave: - pm_runtime_put(sh_chan->dev); + chan->private = NULL; return ret; } @@ -417,7 +444,6 @@ static void sh_dmae_free_chan_resources(struct dma_chan *chan) struct sh_dmae_chan *sh_chan = to_sh_chan(chan); struct sh_desc *desc, *_desc; LIST_HEAD(list); - int descs = sh_chan->descs_allocated; /* Protect against ISR */ spin_lock_irq(&sh_chan->desc_lock); @@ -437,15 +463,12 @@ static void sh_dmae_free_chan_resources(struct dma_chan *chan) chan->private = NULL; } - spin_lock_bh(&sh_chan->desc_lock); + spin_lock_irq(&sh_chan->desc_lock); list_splice_init(&sh_chan->ld_free, &list); sh_chan->descs_allocated = 0; - spin_unlock_bh(&sh_chan->desc_lock); - - if (descs > 0) - pm_runtime_put(sh_chan->dev); + spin_unlock_irq(&sh_chan->desc_lock); list_for_each_entry_safe(desc, _desc, &list, node) kfree(desc); @@ -534,6 +557,7 @@ static struct dma_async_tx_descriptor *sh_dmae_prep_sg(struct sh_dmae_chan *sh_c struct sh_desc *first = NULL, *new = NULL /* compiler... */; LIST_HEAD(tx_list); int chunks = 0; + unsigned long irq_flags; int i; if (!sg_len) @@ -544,7 +568,7 @@ static struct dma_async_tx_descriptor *sh_dmae_prep_sg(struct sh_dmae_chan *sh_c (SH_DMA_TCR_MAX + 1); /* Have to lock the whole loop to protect against concurrent release */ - spin_lock_bh(&sh_chan->desc_lock); + spin_lock_irqsave(&sh_chan->desc_lock, irq_flags); /* * Chaining: @@ -590,7 +614,7 @@ static struct dma_async_tx_descriptor *sh_dmae_prep_sg(struct sh_dmae_chan *sh_c /* Put them back on the free list, so, they don't get lost */ list_splice_tail(&tx_list, &sh_chan->ld_free); - spin_unlock_bh(&sh_chan->desc_lock); + spin_unlock_irqrestore(&sh_chan->desc_lock, irq_flags); return &first->async_tx; @@ -599,7 +623,7 @@ err_get_desc: new->mark = DESC_IDLE; list_splice(&tx_list, &sh_chan->ld_free); - spin_unlock_bh(&sh_chan->desc_lock); + spin_unlock_irqrestore(&sh_chan->desc_lock, irq_flags); return NULL; } @@ -661,6 +685,7 @@ static int sh_dmae_control(struct dma_chan *chan, enum dma_ctrl_cmd cmd, unsigned long arg) { struct sh_dmae_chan *sh_chan = to_sh_chan(chan); + unsigned long flags; /* Only supports DMA_TERMINATE_ALL */ if (cmd != DMA_TERMINATE_ALL) @@ -669,7 +694,7 @@ static int sh_dmae_control(struct dma_chan *chan, enum dma_ctrl_cmd cmd, if (!chan) return -EINVAL; - spin_lock_bh(&sh_chan->desc_lock); + spin_lock_irqsave(&sh_chan->desc_lock, flags); dmae_halt(sh_chan); if (!list_empty(&sh_chan->ld_queue)) { @@ -678,9 +703,8 @@ static int sh_dmae_control(struct dma_chan *chan, enum dma_ctrl_cmd cmd, struct sh_desc, node); desc->partial = (desc->hw.tcr - sh_dmae_readl(sh_chan, TCR)) << sh_chan->xmit_shift; - } - spin_unlock_bh(&sh_chan->desc_lock); + spin_unlock_irqrestore(&sh_chan->desc_lock, flags); sh_dmae_chan_ld_cleanup(sh_chan, true); @@ -695,8 +719,9 @@ static dma_async_tx_callback __ld_cleanup(struct sh_dmae_chan *sh_chan, bool all dma_cookie_t cookie = 0; dma_async_tx_callback callback = NULL; void *param = NULL; + unsigned long flags; - spin_lock_bh(&sh_chan->desc_lock); + spin_lock_irqsave(&sh_chan->desc_lock, flags); list_for_each_entry_safe(desc, _desc, &sh_chan->ld_queue, node) { struct dma_async_tx_descriptor *tx = &desc->async_tx; @@ -762,7 +787,13 @@ static dma_async_tx_callback __ld_cleanup(struct sh_dmae_chan *sh_chan, bool all async_tx_test_ack(&desc->async_tx)) || all) { /* Remove from ld_queue list */ desc->mark = DESC_IDLE; + list_move(&desc->node, &sh_chan->ld_free); + + if (list_empty(&sh_chan->ld_queue)) { + dev_dbg(sh_chan->dev, "Bring down channel %d\n", sh_chan->id); + pm_runtime_put(sh_chan->dev); + } } } @@ -773,7 +804,7 @@ static dma_async_tx_callback __ld_cleanup(struct sh_dmae_chan *sh_chan, bool all */ sh_chan->completed_cookie = sh_chan->common.cookie; - spin_unlock_bh(&sh_chan->desc_lock); + spin_unlock_irqrestore(&sh_chan->desc_lock, flags); if (callback) callback(param); @@ -792,14 +823,14 @@ static void sh_dmae_chan_ld_cleanup(struct sh_dmae_chan *sh_chan, bool all) ; } +/* Called under spin_lock_irq(&sh_chan->desc_lock) */ static void sh_chan_xfer_ld_queue(struct sh_dmae_chan *sh_chan) { struct sh_desc *desc; - spin_lock_bh(&sh_chan->desc_lock); /* DMA work check */ if (dmae_is_busy(sh_chan)) - goto sh_chan_xfer_ld_queue_end; + return; /* Find the first not transferred descriptor */ list_for_each_entry(desc, &sh_chan->ld_queue, node) @@ -812,15 +843,18 @@ static void sh_chan_xfer_ld_queue(struct sh_dmae_chan *sh_chan) dmae_start(sh_chan); break; } - -sh_chan_xfer_ld_queue_end: - spin_unlock_bh(&sh_chan->desc_lock); } static void sh_dmae_memcpy_issue_pending(struct dma_chan *chan) { struct sh_dmae_chan *sh_chan = to_sh_chan(chan); - sh_chan_xfer_ld_queue(sh_chan); + + spin_lock_irq(&sh_chan->desc_lock); + if (sh_chan->pm_state == DMAE_PM_ESTABLISHED) + sh_chan_xfer_ld_queue(sh_chan); + else + sh_chan->pm_state = DMAE_PM_PENDING; + spin_unlock_irq(&sh_chan->desc_lock); } static enum dma_status sh_dmae_tx_status(struct dma_chan *chan, @@ -831,6 +865,7 @@ static enum dma_status sh_dmae_tx_status(struct dma_chan *chan, dma_cookie_t last_used; dma_cookie_t last_complete; enum dma_status status; + unsigned long flags; sh_dmae_chan_ld_cleanup(sh_chan, false); @@ -841,7 +876,7 @@ static enum dma_status sh_dmae_tx_status(struct dma_chan *chan, BUG_ON(last_complete < 0); dma_set_tx_state(txstate, last_complete, last_used, 0); - spin_lock_bh(&sh_chan->desc_lock); + spin_lock_irqsave(&sh_chan->desc_lock, flags); status = dma_async_is_complete(cookie, last_complete, last_used); @@ -859,7 +894,7 @@ static enum dma_status sh_dmae_tx_status(struct dma_chan *chan, } } - spin_unlock_bh(&sh_chan->desc_lock); + spin_unlock_irqrestore(&sh_chan->desc_lock, flags); return status; } @@ -912,6 +947,12 @@ static bool sh_dmae_reset(struct sh_dmae_device *shdev) list_splice_init(&sh_chan->ld_queue, &dl); + if (!list_empty(&dl)) { + dev_dbg(sh_chan->dev, "Bring down channel %d\n", sh_chan->id); + pm_runtime_put(sh_chan->dev); + } + sh_chan->pm_state = DMAE_PM_ESTABLISHED; + spin_unlock(&sh_chan->desc_lock); /* Complete all */ @@ -952,7 +993,7 @@ static void dmae_do_tasklet(unsigned long data) u32 sar_buf = sh_dmae_readl(sh_chan, SAR); u32 dar_buf = sh_dmae_readl(sh_chan, DAR); - spin_lock(&sh_chan->desc_lock); + spin_lock_irq(&sh_chan->desc_lock); list_for_each_entry(desc, &sh_chan->ld_queue, node) { if (desc->mark == DESC_SUBMITTED && ((desc->direction == DMA_FROM_DEVICE && @@ -965,10 +1006,10 @@ static void dmae_do_tasklet(unsigned long data) break; } } - spin_unlock(&sh_chan->desc_lock); - /* Next desc */ sh_chan_xfer_ld_queue(sh_chan); + spin_unlock_irq(&sh_chan->desc_lock); + sh_dmae_chan_ld_cleanup(sh_chan, false); } @@ -1036,7 +1077,9 @@ static int __devinit sh_dmae_chan_probe(struct sh_dmae_device *shdev, int id, return -ENOMEM; } - /* copy struct dma_device */ + new_sh_chan->pm_state = DMAE_PM_ESTABLISHED; + + /* reference struct dma_device */ new_sh_chan->common.device = &shdev->common; new_sh_chan->dev = shdev->common.dev; diff --git a/drivers/dma/shdma.h b/drivers/dma/shdma.h index dc56576f9fd..2b55a276dc5 100644 --- a/drivers/dma/shdma.h +++ b/drivers/dma/shdma.h @@ -23,6 +23,12 @@ struct device; +enum dmae_pm_state { + DMAE_PM_ESTABLISHED, + DMAE_PM_BUSY, + DMAE_PM_PENDING, +}; + struct sh_dmae_chan { dma_cookie_t completed_cookie; /* The maximum cookie completed */ spinlock_t desc_lock; /* Descriptor operation lock */ @@ -38,6 +44,7 @@ struct sh_dmae_chan { u32 __iomem *base; char dev_id[16]; /* unique name per DMAC of channel */ int pm_error; + enum dmae_pm_state pm_state; }; struct sh_dmae_device { diff --git a/drivers/dma/ste_dma40.c b/drivers/dma/ste_dma40.c index 467e4dcb20a..13259cad0ce 100644 --- a/drivers/dma/ste_dma40.c +++ b/drivers/dma/ste_dma40.c @@ -9,6 +9,7 @@ #include <linux/dma-mapping.h> #include <linux/kernel.h> #include <linux/slab.h> +#include <linux/export.h> #include <linux/dmaengine.h> #include <linux/platform_device.h> #include <linux/clk.h> diff --git a/drivers/dma/timb_dma.c b/drivers/dma/timb_dma.c index f69f90a6187..a4a398f2ef6 100644 --- a/drivers/dma/timb_dma.c +++ b/drivers/dma/timb_dma.c @@ -753,7 +753,7 @@ static int __devinit td_probe(struct platform_device *pdev) INIT_LIST_HEAD(&td->dma.channels); - for (i = 0; i < pdata->nr_channels; i++, td->dma.chancnt++) { + for (i = 0; i < pdata->nr_channels; i++) { struct timb_dma_chan *td_chan = &td->channels[i]; struct timb_dma_platform_data_channel *pchan = pdata->channels + i; @@ -762,12 +762,11 @@ static int __devinit td_probe(struct platform_device *pdev) if ((i % 2) == pchan->rx) { dev_err(&pdev->dev, "Wrong channel configuration\n"); err = -EINVAL; - goto err_tasklet_kill; + goto err_free_irq; } td_chan->chan.device = &td->dma; td_chan->chan.cookie = 1; - td_chan->chan.chan_id = i; spin_lock_init(&td_chan->lock); INIT_LIST_HEAD(&td_chan->active_list); INIT_LIST_HEAD(&td_chan->queue); |