diff options
Diffstat (limited to 'drivers/dma/ioat')
| -rw-r--r-- | drivers/dma/ioat/dma.c | 48 | ||||
| -rw-r--r-- | drivers/dma/ioat/dma.h | 11 | ||||
| -rw-r--r-- | drivers/dma/ioat/dma_v2.c | 70 | ||||
| -rw-r--r-- | drivers/dma/ioat/dma_v2.h | 6 | ||||
| -rw-r--r-- | drivers/dma/ioat/dma_v3.c | 64 | ||||
| -rw-r--r-- | drivers/dma/ioat/registers.h | 2 |
6 files changed, 91 insertions, 110 deletions
diff --git a/drivers/dma/ioat/dma.c b/drivers/dma/ioat/dma.c index dcc4ab78b32..af14c9a5b8d 100644 --- a/drivers/dma/ioat/dma.c +++ b/drivers/dma/ioat/dma.c @@ -71,7 +71,7 @@ static irqreturn_t ioat_dma_do_interrupt(int irq, void *data) } attnstatus = readl(instance->reg_base + IOAT_ATTNSTATUS_OFFSET); - for_each_bit(bit, &attnstatus, BITS_PER_LONG) { + for_each_set_bit(bit, &attnstatus, BITS_PER_LONG) { chan = ioat_chan_by_index(instance, bit); tasklet_schedule(&chan->cleanup_task); } @@ -94,16 +94,12 @@ static irqreturn_t ioat_dma_do_interrupt_msix(int irq, void *data) return IRQ_HANDLED; } -static void ioat1_cleanup_tasklet(unsigned long data); - /* common channel initialization */ -void ioat_init_channel(struct ioatdma_device *device, - struct ioat_chan_common *chan, int idx, - void (*timer_fn)(unsigned long), - void (*tasklet)(unsigned long), - unsigned long ioat) +void ioat_init_channel(struct ioatdma_device *device, struct ioat_chan_common *chan, int idx) { struct dma_device *dma = &device->common; + struct dma_chan *c = &chan->common; + unsigned long data = (unsigned long) c; chan->device = device; chan->reg_base = device->reg_base + (0x80 * (idx + 1)); @@ -112,14 +108,12 @@ void ioat_init_channel(struct ioatdma_device *device, list_add_tail(&chan->common.device_node, &dma->channels); device->idx[idx] = chan; init_timer(&chan->timer); - chan->timer.function = timer_fn; - chan->timer.data = ioat; - tasklet_init(&chan->cleanup_task, tasklet, ioat); + chan->timer.function = device->timer_fn; + chan->timer.data = data; + tasklet_init(&chan->cleanup_task, device->cleanup_fn, data); tasklet_disable(&chan->cleanup_task); } -static void ioat1_timer_event(unsigned long data); - /** * ioat1_dma_enumerate_channels - find and initialize the device's channels * @device: the device to be enumerated @@ -155,10 +149,7 @@ static int ioat1_enumerate_channels(struct ioatdma_device *device) if (!ioat) break; - ioat_init_channel(device, &ioat->base, i, - ioat1_timer_event, - ioat1_cleanup_tasklet, - (unsigned long) ioat); + ioat_init_channel(device, &ioat->base, i); ioat->xfercap = xfercap; spin_lock_init(&ioat->desc_lock); INIT_LIST_HEAD(&ioat->free_desc); @@ -532,12 +523,12 @@ ioat1_dma_prep_memcpy(struct dma_chan *c, dma_addr_t dma_dest, return &desc->txd; } -static void ioat1_cleanup_tasklet(unsigned long data) +static void ioat1_cleanup_event(unsigned long data) { - struct ioat_dma_chan *chan = (void *)data; + struct ioat_dma_chan *ioat = to_ioat_chan((void *) data); - ioat1_cleanup(chan); - writew(IOAT_CHANCTRL_RUN, chan->base.reg_base + IOAT_CHANCTRL_OFFSET); + ioat1_cleanup(ioat); + writew(IOAT_CHANCTRL_RUN, ioat->base.reg_base + IOAT_CHANCTRL_OFFSET); } void ioat_dma_unmap(struct ioat_chan_common *chan, enum dma_ctrl_flags flags, @@ -687,7 +678,7 @@ static void ioat1_cleanup(struct ioat_dma_chan *ioat) static void ioat1_timer_event(unsigned long data) { - struct ioat_dma_chan *ioat = (void *) data; + struct ioat_dma_chan *ioat = to_ioat_chan((void *) data); struct ioat_chan_common *chan = &ioat->base; dev_dbg(to_dev(chan), "%s: state: %lx\n", __func__, chan->state); @@ -734,16 +725,17 @@ static void ioat1_timer_event(unsigned long data) spin_unlock_bh(&chan->cleanup_lock); } -static enum dma_status -ioat1_dma_is_complete(struct dma_chan *c, dma_cookie_t cookie, +enum dma_status +ioat_is_dma_complete(struct dma_chan *c, dma_cookie_t cookie, dma_cookie_t *done, dma_cookie_t *used) { - struct ioat_dma_chan *ioat = to_ioat_chan(c); + struct ioat_chan_common *chan = to_chan_common(c); + struct ioatdma_device *device = chan->device; if (ioat_is_complete(c, cookie, done, used) == DMA_SUCCESS) return DMA_SUCCESS; - ioat1_cleanup(ioat); + device->cleanup_fn((unsigned long) c); return ioat_is_complete(c, cookie, done, used); } @@ -1199,12 +1191,14 @@ int __devinit ioat1_dma_probe(struct ioatdma_device *device, int dca) device->intr_quirk = ioat1_intr_quirk; device->enumerate_channels = ioat1_enumerate_channels; device->self_test = ioat_dma_self_test; + device->timer_fn = ioat1_timer_event; + device->cleanup_fn = ioat1_cleanup_event; dma = &device->common; dma->device_prep_dma_memcpy = ioat1_dma_prep_memcpy; dma->device_issue_pending = ioat1_dma_memcpy_issue_pending; dma->device_alloc_chan_resources = ioat1_dma_alloc_chan_resources; dma->device_free_chan_resources = ioat1_dma_free_chan_resources; - dma->device_is_tx_complete = ioat1_dma_is_complete; + dma->device_is_tx_complete = ioat_is_dma_complete; err = ioat_probe(device); if (err) diff --git a/drivers/dma/ioat/dma.h b/drivers/dma/ioat/dma.h index bbc3e78ef33..4f747a25407 100644 --- a/drivers/dma/ioat/dma.h +++ b/drivers/dma/ioat/dma.h @@ -61,7 +61,7 @@ * @intr_quirk: interrupt setup quirk (for ioat_v1 devices) * @enumerate_channels: hw version specific channel enumeration * @reset_hw: hw version specific channel (re)initialization - * @cleanup_tasklet: select between the v2 and v3 cleanup routines + * @cleanup_fn: select between the v2 and v3 cleanup routines * @timer_fn: select between the v2 and v3 timer watchdog routines * @self_test: hardware version specific self test for each supported op type * @@ -80,7 +80,7 @@ struct ioatdma_device { void (*intr_quirk)(struct ioatdma_device *device); int (*enumerate_channels)(struct ioatdma_device *device); int (*reset_hw)(struct ioat_chan_common *chan); - void (*cleanup_tasklet)(unsigned long data); + void (*cleanup_fn)(unsigned long data); void (*timer_fn)(unsigned long data); int (*self_test)(struct ioatdma_device *device); }; @@ -337,10 +337,9 @@ struct dca_provider * __devinit ioat_dca_init(struct pci_dev *pdev, void __iomem *iobase); unsigned long ioat_get_current_completion(struct ioat_chan_common *chan); void ioat_init_channel(struct ioatdma_device *device, - struct ioat_chan_common *chan, int idx, - void (*timer_fn)(unsigned long), - void (*tasklet)(unsigned long), - unsigned long ioat); + struct ioat_chan_common *chan, int idx); +enum dma_status ioat_is_dma_complete(struct dma_chan *c, dma_cookie_t cookie, + dma_cookie_t *done, dma_cookie_t *used); void ioat_dma_unmap(struct ioat_chan_common *chan, enum dma_ctrl_flags flags, size_t len, struct ioat_dma_descriptor *hw); bool ioat_cleanup_preamble(struct ioat_chan_common *chan, diff --git a/drivers/dma/ioat/dma_v2.c b/drivers/dma/ioat/dma_v2.c index 5cc37afe2bc..1ed5d66d7dc 100644 --- a/drivers/dma/ioat/dma_v2.c +++ b/drivers/dma/ioat/dma_v2.c @@ -51,48 +51,40 @@ MODULE_PARM_DESC(ioat_ring_max_alloc_order, void __ioat2_issue_pending(struct ioat2_dma_chan *ioat) { - void * __iomem reg_base = ioat->base.reg_base; + struct ioat_chan_common *chan = &ioat->base; - ioat->pending = 0; ioat->dmacount += ioat2_ring_pending(ioat); ioat->issued = ioat->head; /* make descriptor updates globally visible before notifying channel */ wmb(); - writew(ioat->dmacount, reg_base + IOAT_CHAN_DMACOUNT_OFFSET); - dev_dbg(to_dev(&ioat->base), + writew(ioat->dmacount, chan->reg_base + IOAT_CHAN_DMACOUNT_OFFSET); + dev_dbg(to_dev(chan), "%s: head: %#x tail: %#x issued: %#x count: %#x\n", __func__, ioat->head, ioat->tail, ioat->issued, ioat->dmacount); } -void ioat2_issue_pending(struct dma_chan *chan) +void ioat2_issue_pending(struct dma_chan *c) { - struct ioat2_dma_chan *ioat = to_ioat2_chan(chan); + struct ioat2_dma_chan *ioat = to_ioat2_chan(c); - spin_lock_bh(&ioat->ring_lock); - if (ioat->pending == 1) + if (ioat2_ring_pending(ioat)) { + spin_lock_bh(&ioat->ring_lock); __ioat2_issue_pending(ioat); - spin_unlock_bh(&ioat->ring_lock); + spin_unlock_bh(&ioat->ring_lock); + } } /** * ioat2_update_pending - log pending descriptors * @ioat: ioat2+ channel * - * set pending to '1' unless pending is already set to '2', pending == 2 - * indicates that submission is temporarily blocked due to an in-flight - * reset. If we are already above the ioat_pending_level threshold then - * just issue pending. - * - * called with ring_lock held + * Check if the number of unsubmitted descriptors has exceeded the + * watermark. Called with ring_lock held */ static void ioat2_update_pending(struct ioat2_dma_chan *ioat) { - if (unlikely(ioat->pending == 2)) - return; - else if (ioat2_ring_pending(ioat) > ioat_pending_level) + if (ioat2_ring_pending(ioat) > ioat_pending_level) __ioat2_issue_pending(ioat); - else - ioat->pending = 1; } static void __ioat2_start_null_desc(struct ioat2_dma_chan *ioat) @@ -166,7 +158,7 @@ static void __cleanup(struct ioat2_dma_chan *ioat, unsigned long phys_complete) seen_current = true; } ioat->tail += i; - BUG_ON(!seen_current); /* no active descs have written a completion? */ + BUG_ON(active && !seen_current); /* no active descs have written a completion? */ chan->last_completion = phys_complete; if (ioat->head == ioat->tail) { @@ -207,9 +199,9 @@ static void ioat2_cleanup(struct ioat2_dma_chan *ioat) spin_unlock_bh(&chan->cleanup_lock); } -void ioat2_cleanup_tasklet(unsigned long data) +void ioat2_cleanup_event(unsigned long data) { - struct ioat2_dma_chan *ioat = (void *) data; + struct ioat2_dma_chan *ioat = to_ioat2_chan((void *) data); ioat2_cleanup(ioat); writew(IOAT_CHANCTRL_RUN, ioat->base.reg_base + IOAT_CHANCTRL_OFFSET); @@ -291,7 +283,7 @@ static void ioat2_restart_channel(struct ioat2_dma_chan *ioat) void ioat2_timer_event(unsigned long data) { - struct ioat2_dma_chan *ioat = (void *) data; + struct ioat2_dma_chan *ioat = to_ioat2_chan((void *) data); struct ioat_chan_common *chan = &ioat->base; spin_lock_bh(&chan->cleanup_lock); @@ -397,10 +389,7 @@ int ioat2_enumerate_channels(struct ioatdma_device *device) if (!ioat) break; - ioat_init_channel(device, &ioat->base, i, - device->timer_fn, - device->cleanup_tasklet, - (unsigned long) ioat); + ioat_init_channel(device, &ioat->base, i); ioat->xfercap_log = xfercap_log; spin_lock_init(&ioat->ring_lock); if (device->reset_hw(&ioat->base)) { @@ -546,7 +535,6 @@ int ioat2_alloc_chan_resources(struct dma_chan *c) ioat->head = 0; ioat->issued = 0; ioat->tail = 0; - ioat->pending = 0; ioat->alloc_order = order; spin_unlock_bh(&ioat->ring_lock); @@ -701,7 +689,7 @@ int ioat2_alloc_and_lock(u16 *idx, struct ioat2_dma_chan *ioat, int num_descs) mod_timer(&chan->timer, jiffies + COMPLETION_TIMEOUT); spin_unlock_bh(&chan->cleanup_lock); - device->timer_fn((unsigned long) ioat); + device->timer_fn((unsigned long) &chan->common); } else spin_unlock_bh(&chan->cleanup_lock); return -ENOMEM; @@ -785,7 +773,7 @@ void ioat2_free_chan_resources(struct dma_chan *c) tasklet_disable(&chan->cleanup_task); del_timer_sync(&chan->timer); - device->cleanup_tasklet((unsigned long) ioat); + device->cleanup_fn((unsigned long) c); device->reset_hw(chan); spin_lock_bh(&ioat->ring_lock); @@ -815,25 +803,9 @@ void ioat2_free_chan_resources(struct dma_chan *c) chan->last_completion = 0; chan->completion_dma = 0; - ioat->pending = 0; ioat->dmacount = 0; } -enum dma_status -ioat2_is_complete(struct dma_chan *c, dma_cookie_t cookie, - dma_cookie_t *done, dma_cookie_t *used) -{ - struct ioat2_dma_chan *ioat = to_ioat2_chan(c); - struct ioatdma_device *device = ioat->base.device; - - if (ioat_is_complete(c, cookie, done, used) == DMA_SUCCESS) - return DMA_SUCCESS; - - device->cleanup_tasklet((unsigned long) ioat); - - return ioat_is_complete(c, cookie, done, used); -} - static ssize_t ring_size_show(struct dma_chan *c, char *page) { struct ioat2_dma_chan *ioat = to_ioat2_chan(c); @@ -874,7 +846,7 @@ int __devinit ioat2_dma_probe(struct ioatdma_device *device, int dca) device->enumerate_channels = ioat2_enumerate_channels; device->reset_hw = ioat2_reset_hw; - device->cleanup_tasklet = ioat2_cleanup_tasklet; + device->cleanup_fn = ioat2_cleanup_event; device->timer_fn = ioat2_timer_event; device->self_test = ioat_dma_self_test; dma = &device->common; @@ -882,7 +854,7 @@ int __devinit ioat2_dma_probe(struct ioatdma_device *device, int dca) dma->device_issue_pending = ioat2_issue_pending; dma->device_alloc_chan_resources = ioat2_alloc_chan_resources; dma->device_free_chan_resources = ioat2_free_chan_resources; - dma->device_is_tx_complete = ioat2_is_complete; + dma->device_is_tx_complete = ioat_is_dma_complete; err = ioat_probe(device); if (err) diff --git a/drivers/dma/ioat/dma_v2.h b/drivers/dma/ioat/dma_v2.h index 3afad8da43c..ef2871fd786 100644 --- a/drivers/dma/ioat/dma_v2.h +++ b/drivers/dma/ioat/dma_v2.h @@ -47,7 +47,6 @@ extern int ioat_ring_alloc_order; * @head: allocated index * @issued: hardware notification point * @tail: cleanup index - * @pending: lock free indicator for issued != head * @dmacount: identical to 'head' except for occasionally resetting to zero * @alloc_order: log2 of the number of allocated descriptors * @ring: software ring buffer implementation of hardware ring @@ -61,7 +60,6 @@ struct ioat2_dma_chan { u16 tail; u16 dmacount; u16 alloc_order; - int pending; struct ioat_ring_ent **ring; spinlock_t ring_lock; }; @@ -178,12 +176,10 @@ ioat2_dma_prep_memcpy_lock(struct dma_chan *c, dma_addr_t dma_dest, void ioat2_issue_pending(struct dma_chan *chan); int ioat2_alloc_chan_resources(struct dma_chan *c); void ioat2_free_chan_resources(struct dma_chan *c); -enum dma_status ioat2_is_complete(struct dma_chan *c, dma_cookie_t cookie, - dma_cookie_t *done, dma_cookie_t *used); void __ioat2_restart_chan(struct ioat2_dma_chan *ioat); bool reshape_ring(struct ioat2_dma_chan *ioat, int order); void __ioat2_issue_pending(struct ioat2_dma_chan *ioat); -void ioat2_cleanup_tasklet(unsigned long data); +void ioat2_cleanup_event(unsigned long data); void ioat2_timer_event(unsigned long data); int ioat2_quiesce(struct ioat_chan_common *chan, unsigned long tmo); int ioat2_reset_sync(struct ioat_chan_common *chan, unsigned long tmo); diff --git a/drivers/dma/ioat/dma_v3.c b/drivers/dma/ioat/dma_v3.c index 9908c9e94b2..26febc56dab 100644 --- a/drivers/dma/ioat/dma_v3.c +++ b/drivers/dma/ioat/dma_v3.c @@ -293,17 +293,25 @@ static void __cleanup(struct ioat2_dma_chan *ioat, unsigned long phys_complete) } } ioat->tail += i; - BUG_ON(!seen_current); /* no active descs have written a completion? */ + BUG_ON(active && !seen_current); /* no active descs have written a completion? */ chan->last_completion = phys_complete; - if (ioat->head == ioat->tail) { + + active = ioat2_ring_active(ioat); + if (active == 0) { dev_dbg(to_dev(chan), "%s: cancel completion timeout\n", __func__); clear_bit(IOAT_COMPLETION_PENDING, &chan->state); mod_timer(&chan->timer, jiffies + IDLE_TIMEOUT); } + /* 5 microsecond delay per pending descriptor */ + writew(min((5 * active), IOAT_INTRDELAY_MASK), + chan->device->reg_base + IOAT_INTRDELAY_OFFSET); } -static void ioat3_cleanup(struct ioat2_dma_chan *ioat) +/* try to cleanup, but yield (via spin_trylock) to incoming submissions + * with the expectation that we will immediately poll again shortly + */ +static void ioat3_cleanup_poll(struct ioat2_dma_chan *ioat) { struct ioat_chan_common *chan = &ioat->base; unsigned long phys_complete; @@ -329,29 +337,41 @@ static void ioat3_cleanup(struct ioat2_dma_chan *ioat) spin_unlock_bh(&chan->cleanup_lock); } -static void ioat3_cleanup_tasklet(unsigned long data) +/* run cleanup now because we already delayed the interrupt via INTRDELAY */ +static void ioat3_cleanup_sync(struct ioat2_dma_chan *ioat) +{ + struct ioat_chan_common *chan = &ioat->base; + unsigned long phys_complete; + + prefetch(chan->completion); + + spin_lock_bh(&chan->cleanup_lock); + if (!ioat_cleanup_preamble(chan, &phys_complete)) { + spin_unlock_bh(&chan->cleanup_lock); + return; + } + spin_lock_bh(&ioat->ring_lock); + + __cleanup(ioat, phys_complete); + + spin_unlock_bh(&ioat->ring_lock); + spin_unlock_bh(&chan->cleanup_lock); +} + +static void ioat3_cleanup_event(unsigned long data) { - struct ioat2_dma_chan *ioat = (void *) data; + struct ioat2_dma_chan *ioat = to_ioat2_chan((void *) data); - ioat3_cleanup(ioat); - writew(IOAT_CHANCTRL_RUN | IOAT3_CHANCTRL_COMPL_DCA_EN, - ioat->base.reg_base + IOAT_CHANCTRL_OFFSET); + ioat3_cleanup_sync(ioat); + writew(IOAT_CHANCTRL_RUN, ioat->base.reg_base + IOAT_CHANCTRL_OFFSET); } static void ioat3_restart_channel(struct ioat2_dma_chan *ioat) { struct ioat_chan_common *chan = &ioat->base; unsigned long phys_complete; - u32 status; - - status = ioat_chansts(chan); - if (is_ioat_active(status) || is_ioat_idle(status)) - ioat_suspend(chan); - while (is_ioat_active(status) || is_ioat_idle(status)) { - status = ioat_chansts(chan); - cpu_relax(); - } + ioat2_quiesce(chan, 0); if (ioat_cleanup_preamble(chan, &phys_complete)) __cleanup(ioat, phys_complete); @@ -360,7 +380,7 @@ static void ioat3_restart_channel(struct ioat2_dma_chan *ioat) static void ioat3_timer_event(unsigned long data) { - struct ioat2_dma_chan *ioat = (void *) data; + struct ioat2_dma_chan *ioat = to_ioat2_chan((void *) data); struct ioat_chan_common *chan = &ioat->base; spin_lock_bh(&chan->cleanup_lock); @@ -426,7 +446,7 @@ ioat3_is_complete(struct dma_chan *c, dma_cookie_t cookie, if (ioat_is_complete(c, cookie, done, used) == DMA_SUCCESS) return DMA_SUCCESS; - ioat3_cleanup(ioat); + ioat3_cleanup_poll(ioat); return ioat_is_complete(c, cookie, done, used); } @@ -1239,11 +1259,11 @@ int __devinit ioat3_dma_probe(struct ioatdma_device *device, int dca) if (is_raid_device) { dma->device_is_tx_complete = ioat3_is_complete; - device->cleanup_tasklet = ioat3_cleanup_tasklet; + device->cleanup_fn = ioat3_cleanup_event; device->timer_fn = ioat3_timer_event; } else { - dma->device_is_tx_complete = ioat2_is_complete; - device->cleanup_tasklet = ioat2_cleanup_tasklet; + dma->device_is_tx_complete = ioat_is_dma_complete; + device->cleanup_fn = ioat2_cleanup_event; device->timer_fn = ioat2_timer_event; } diff --git a/drivers/dma/ioat/registers.h b/drivers/dma/ioat/registers.h index e8ae63baf58..1391798542b 100644 --- a/drivers/dma/ioat/registers.h +++ b/drivers/dma/ioat/registers.h @@ -60,7 +60,7 @@ #define IOAT_PERPORTOFFSET_OFFSET 0x0A /* 16-bit */ #define IOAT_INTRDELAY_OFFSET 0x0C /* 16-bit */ -#define IOAT_INTRDELAY_INT_DELAY_MASK 0x3FFF /* Interrupt Delay Time */ +#define IOAT_INTRDELAY_MASK 0x3FFF /* Interrupt Delay Time */ #define IOAT_INTRDELAY_COALESE_SUPPORT 0x8000 /* Interrupt Coalescing Supported */ #define IOAT_DEVICE_STATUS_OFFSET 0x0E /* 16-bit */ |