From 07f2211e4fbce6990722d78c4f04225da9c0e9cf Mon Sep 17 00:00:00 2001
From: Dan Williams <dan.j.williams@intel.com>
Date: Mon, 5 Jan 2009 17:14:31 -0700
Subject: dmaengine: remove dependency on async_tx
async_tx.ko is a consumer of dma channels. A circular dependency arises
if modules in drivers/dma rely on common code in async_tx.ko. It
prevents either module from being unloaded.
Move dma_wait_for_async_tx and async_tx_run_dependencies to dmaeninge.o
where they should have been from the beginning.
Reviewed-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Dan Williams <dan.j.williams@intel.com>
---
crypto/async_tx/async_tx.c | 75 ----------------------------------------------
1 file changed, 75 deletions(-)
(limited to 'crypto/async_tx')
diff --git a/crypto/async_tx/async_tx.c b/crypto/async_tx/async_tx.c
index dcbf1be149f..8cfac182165 100644
--- a/crypto/async_tx/async_tx.c
+++ b/crypto/async_tx/async_tx.c
@@ -72,81 +72,6 @@ void async_tx_issue_pending_all(void)
}
EXPORT_SYMBOL_GPL(async_tx_issue_pending_all);
-/* dma_wait_for_async_tx - spin wait for a transcation to complete
- * @tx: transaction to wait on
- */
-enum dma_status
-dma_wait_for_async_tx(struct dma_async_tx_descriptor *tx)
-{
- enum dma_status status;
- struct dma_async_tx_descriptor *iter;
- struct dma_async_tx_descriptor *parent;
-
- if (!tx)
- return DMA_SUCCESS;
-
- /* poll through the dependency chain, return when tx is complete */
- do {
- iter = tx;
-
- /* find the root of the unsubmitted dependency chain */
- do {
- parent = iter->parent;
- if (!parent)
- break;
- else
- iter = parent;
- } while (parent);
-
- /* there is a small window for ->parent == NULL and
- * ->cookie == -EBUSY
- */
- while (iter->cookie == -EBUSY)
- cpu_relax();
-
- status = dma_sync_wait(iter->chan, iter->cookie);
- } while (status == DMA_IN_PROGRESS || (iter != tx));
-
- return status;
-}
-EXPORT_SYMBOL_GPL(dma_wait_for_async_tx);
-
-/* async_tx_run_dependencies - helper routine for dma drivers to process
- * (start) dependent operations on their target channel
- * @tx: transaction with dependencies
- */
-void async_tx_run_dependencies(struct dma_async_tx_descriptor *tx)
-{
- struct dma_async_tx_descriptor *dep = tx->next;
- struct dma_async_tx_descriptor *dep_next;
- struct dma_chan *chan;
-
- if (!dep)
- return;
-
- chan = dep->chan;
-
- /* keep submitting up until a channel switch is detected
- * in that case we will be called again as a result of
- * processing the interrupt from async_tx_channel_switch
- */
- for (; dep; dep = dep_next) {
- spin_lock_bh(&dep->lock);
- dep->parent = NULL;
- dep_next = dep->next;
- if (dep_next && dep_next->chan == chan)
- dep->next = NULL; /* ->next will be submitted */
- else
- dep_next = NULL; /* submit current dep and terminate */
- spin_unlock_bh(&dep->lock);
-
- dep->tx_submit(dep);
- }
-
- chan->device->device_issue_pending(chan);
-}
-EXPORT_SYMBOL_GPL(async_tx_run_dependencies);
-
static void
free_dma_chan_ref(struct rcu_head *rcu)
{
--
cgit v1.2.3-70-g09d2
From 6f49a57aa5a0c6d4e4e27c85f7af6c83325a12d1 Mon Sep 17 00:00:00 2001
From: Dan Williams <dan.j.williams@intel.com>
Date: Tue, 6 Jan 2009 11:38:14 -0700
Subject: dmaengine: up-level reference counting to the module level
Simply, if a client wants any dmaengine channel then prevent all dmaengine
modules from being removed. Once the clients are done re-enable module
removal.
Why?, beyond reducing complication:
1/ Tracking reference counts per-transaction in an efficient manner, as
is currently done, requires a complicated scheme to avoid cache-line
bouncing effects.
2/ Per-transaction ref-counting gives the false impression that a
dma-driver can be gracefully removed ahead of its user (net, md, or
dma-slave)
3/ None of the in-tree dma-drivers talk to hot pluggable hardware, but
if such an engine were built one day we still would not need to notify
clients of remove events. The driver can simply return NULL to a
->prep() request, something that is much easier for a client to handle.
Reviewed-by: Andrew Morton <akpm@linux-foundation.org>
Acked-by: Maciej Sosnowski <maciej.sosnowski@intel.com>
Signed-off-by: Dan Williams <dan.j.williams@intel.com>
---
crypto/async_tx/async_tx.c | 4 -
drivers/dma/dmaengine.c | 205 +++++++++++++++++++++++++++----------------
drivers/dma/dmatest.c | 2 -
drivers/dma/dw_dmac.c | 2 +-
drivers/mmc/host/atmel-mci.c | 4 +-
include/linux/dmaengine.h | 21 -----
include/net/netdma.h | 4 +-
net/ipv4/tcp.c | 1 -
8 files changed, 132 insertions(+), 111 deletions(-)
(limited to 'crypto/async_tx')
diff --git a/crypto/async_tx/async_tx.c b/crypto/async_tx/async_tx.c
index 8cfac182165..43fe4cbe71e 100644
--- a/crypto/async_tx/async_tx.c
+++ b/crypto/async_tx/async_tx.c
@@ -198,8 +198,6 @@ dma_channel_add_remove(struct dma_client *client,
/* add the channel to the generic management list */
master_ref = kmalloc(sizeof(*master_ref), GFP_KERNEL);
if (master_ref) {
- /* keep a reference until async_tx is unloaded */
- dma_chan_get(chan);
init_dma_chan_ref(master_ref, chan);
spin_lock_irqsave(&async_tx_lock, flags);
list_add_tail_rcu(&master_ref->node,
@@ -221,8 +219,6 @@ dma_channel_add_remove(struct dma_client *client,
spin_lock_irqsave(&async_tx_lock, flags);
list_for_each_entry(ref, &async_tx_master_list, node)
if (ref->chan == chan) {
- /* permit backing devices to go away */
- dma_chan_put(ref->chan);
list_del_rcu(&ref->node);
call_rcu(&ref->rcu, free_dma_chan_ref);
found = 1;
diff --git a/drivers/dma/dmaengine.c b/drivers/dma/dmaengine.c
index b9008932a8f..d4d925912c4 100644
--- a/drivers/dma/dmaengine.c
+++ b/drivers/dma/dmaengine.c
@@ -74,6 +74,7 @@
static DEFINE_MUTEX(dma_list_mutex);
static LIST_HEAD(dma_device_list);
static LIST_HEAD(dma_client_list);
+static long dmaengine_ref_count;
/* --- sysfs implementation --- */
@@ -105,19 +106,8 @@ static ssize_t show_bytes_transferred(struct device *dev, struct device_attribut
static ssize_t show_in_use(struct device *dev, struct device_attribute *attr, char *buf)
{
struct dma_chan *chan = to_dma_chan(dev);
- int in_use = 0;
-
- if (unlikely(chan->slow_ref) &&
- atomic_read(&chan->refcount.refcount) > 1)
- in_use = 1;
- else {
- if (local_read(&(per_cpu_ptr(chan->local,
- get_cpu())->refcount)) > 0)
- in_use = 1;
- put_cpu();
- }
- return sprintf(buf, "%d\n", in_use);
+ return sprintf(buf, "%d\n", chan->client_count);
}
static struct device_attribute dma_attrs[] = {
@@ -155,6 +145,78 @@ __dma_chan_satisfies_mask(struct dma_chan *chan, dma_cap_mask_t *want)
return bitmap_equal(want->bits, has.bits, DMA_TX_TYPE_END);
}
+static struct module *dma_chan_to_owner(struct dma_chan *chan)
+{
+ return chan->device->dev->driver->owner;
+}
+
+/**
+ * balance_ref_count - catch up the channel reference count
+ * @chan - channel to balance ->client_count versus dmaengine_ref_count
+ *
+ * balance_ref_count must be called under dma_list_mutex
+ */
+static void balance_ref_count(struct dma_chan *chan)
+{
+ struct module *owner = dma_chan_to_owner(chan);
+
+ while (chan->client_count < dmaengine_ref_count) {
+ __module_get(owner);
+ chan->client_count++;
+ }
+}
+
+/**
+ * dma_chan_get - try to grab a dma channel's parent driver module
+ * @chan - channel to grab
+ *
+ * Must be called under dma_list_mutex
+ */
+static int dma_chan_get(struct dma_chan *chan)
+{
+ int err = -ENODEV;
+ struct module *owner = dma_chan_to_owner(chan);
+
+ if (chan->client_count) {
+ __module_get(owner);
+ err = 0;
+ } else if (try_module_get(owner))
+ err = 0;
+
+ if (err == 0)
+ chan->client_count++;
+
+ /* allocate upon first client reference */
+ if (chan->client_count == 1 && err == 0) {
+ int desc_cnt = chan->device->device_alloc_chan_resources(chan, NULL);
+
+ if (desc_cnt < 0) {
+ err = desc_cnt;
+ chan->client_count = 0;
+ module_put(owner);
+ } else
+ balance_ref_count(chan);
+ }
+
+ return err;
+}
+
+/**
+ * dma_chan_put - drop a reference to a dma channel's parent driver module
+ * @chan - channel to release
+ *
+ * Must be called under dma_list_mutex
+ */
+static void dma_chan_put(struct dma_chan *chan)
+{
+ if (!chan->client_count)
+ return; /* this channel failed alloc_chan_resources */
+ chan->client_count--;
+ module_put(dma_chan_to_owner(chan));
+ if (chan->client_count == 0)
+ chan->device->device_free_chan_resources(chan);
+}
+
/**
* dma_client_chan_alloc - try to allocate channels to a client
* @client: &dma_client
@@ -165,7 +227,6 @@ static void dma_client_chan_alloc(struct dma_client *client)
{
struct dma_device *device;
struct dma_chan *chan;
- int desc; /* allocated descriptor count */
enum dma_state_client ack;
/* Find a channel */
@@ -178,23 +239,16 @@ static void dma_client_chan_alloc(struct dma_client *client)
list_for_each_entry(chan, &device->channels, device_node) {
if (!dma_chan_satisfies_mask(chan, client->cap_mask))
continue;
+ if (!chan->client_count)
+ continue;
+ ack = client->event_callback(client, chan,
+ DMA_RESOURCE_AVAILABLE);
- desc = chan->device->device_alloc_chan_resources(
- chan, client);
- if (desc >= 0) {
- ack = client->event_callback(client,
- chan,
- DMA_RESOURCE_AVAILABLE);
-
- /* we are done once this client rejects
- * an available resource
- */
- if (ack == DMA_ACK) {
- dma_chan_get(chan);
- chan->client_count++;
- } else if (ack == DMA_NAK)
- return;
- }
+ /* we are done once this client rejects
+ * an available resource
+ */
+ if (ack == DMA_NAK)
+ return;
}
}
}
@@ -224,7 +278,6 @@ EXPORT_SYMBOL(dma_sync_wait);
void dma_chan_cleanup(struct kref *kref)
{
struct dma_chan *chan = container_of(kref, struct dma_chan, refcount);
- chan->device->device_free_chan_resources(chan);
kref_put(&chan->device->refcount, dma_async_device_cleanup);
}
EXPORT_SYMBOL(dma_chan_cleanup);
@@ -232,18 +285,12 @@ EXPORT_SYMBOL(dma_chan_cleanup);
static void dma_chan_free_rcu(struct rcu_head *rcu)
{
struct dma_chan *chan = container_of(rcu, struct dma_chan, rcu);
- int bias = 0x7FFFFFFF;
- int i;
- for_each_possible_cpu(i)
- bias -= local_read(&per_cpu_ptr(chan->local, i)->refcount);
- atomic_sub(bias, &chan->refcount.refcount);
+
kref_put(&chan->refcount, dma_chan_cleanup);
}
static void dma_chan_release(struct dma_chan *chan)
{
- atomic_add(0x7FFFFFFF, &chan->refcount.refcount);
- chan->slow_ref = 1;
call_rcu(&chan->rcu, dma_chan_free_rcu);
}
@@ -262,44 +309,37 @@ static void dma_clients_notify_available(void)
mutex_unlock(&dma_list_mutex);
}
-/**
- * dma_chans_notify_available - tell the clients that a channel is going away
- * @chan: channel on its way out
- */
-static void dma_clients_notify_removed(struct dma_chan *chan)
-{
- struct dma_client *client;
- enum dma_state_client ack;
-
- mutex_lock(&dma_list_mutex);
-
- list_for_each_entry(client, &dma_client_list, global_node) {
- ack = client->event_callback(client, chan,
- DMA_RESOURCE_REMOVED);
-
- /* client was holding resources for this channel so
- * free it
- */
- if (ack == DMA_ACK) {
- dma_chan_put(chan);
- chan->client_count--;
- }
- }
-
- mutex_unlock(&dma_list_mutex);
-}
-
/**
* dma_async_client_register - register a &dma_client
* @client: ptr to a client structure with valid 'event_callback' and 'cap_mask'
*/
void dma_async_client_register(struct dma_client *client)
{
+ struct dma_device *device, *_d;
+ struct dma_chan *chan;
+ int err;
+
/* validate client data */
BUG_ON(dma_has_cap(DMA_SLAVE, client->cap_mask) &&
!client->slave);
mutex_lock(&dma_list_mutex);
+ dmaengine_ref_count++;
+
+ /* try to grab channels */
+ list_for_each_entry_safe(device, _d, &dma_device_list, global_node)
+ list_for_each_entry(chan, &device->channels, device_node) {
+ err = dma_chan_get(chan);
+ if (err == -ENODEV) {
+ /* module removed before we could use it */
+ list_del_init(&device->global_node);
+ break;
+ } else if (err)
+ pr_err("dmaengine: failed to get %s: (%d)\n",
+ dev_name(&chan->dev), err);
+ }
+
+
list_add_tail(&client->global_node, &dma_client_list);
mutex_unlock(&dma_list_mutex);
}
@@ -315,23 +355,17 @@ void dma_async_client_unregister(struct dma_client *client)
{
struct dma_device *device;
struct dma_chan *chan;
- enum dma_state_client ack;
if (!client)
return;
mutex_lock(&dma_list_mutex);
- /* free all channels the client is holding */
+ dmaengine_ref_count--;
+ BUG_ON(dmaengine_ref_count < 0);
+ /* drop channel references */
list_for_each_entry(device, &dma_device_list, global_node)
- list_for_each_entry(chan, &device->channels, device_node) {
- ack = client->event_callback(client, chan,
- DMA_RESOURCE_REMOVED);
-
- if (ack == DMA_ACK) {
- dma_chan_put(chan);
- chan->client_count--;
- }
- }
+ list_for_each_entry(chan, &device->channels, device_node)
+ dma_chan_put(chan);
list_del(&client->global_node);
mutex_unlock(&dma_list_mutex);
@@ -423,6 +457,21 @@ int dma_async_device_register(struct dma_device *device)
}
mutex_lock(&dma_list_mutex);
+ if (dmaengine_ref_count)
+ list_for_each_entry(chan, &device->channels, device_node) {
+ /* if clients are already waiting for channels we need
+ * to take references on their behalf
+ */
+ if (dma_chan_get(chan) == -ENODEV) {
+ /* note we can only get here for the first
+ * channel as the remaining channels are
+ * guaranteed to get a reference
+ */
+ rc = -ENODEV;
+ mutex_unlock(&dma_list_mutex);
+ goto err_out;
+ }
+ }
list_add_tail(&device->global_node, &dma_device_list);
mutex_unlock(&dma_list_mutex);
@@ -456,7 +505,7 @@ static void dma_async_device_cleanup(struct kref *kref)
}
/**
- * dma_async_device_unregister - unregisters DMA devices
+ * dma_async_device_unregister - unregister a DMA device
* @device: &dma_device
*/
void dma_async_device_unregister(struct dma_device *device)
@@ -468,7 +517,9 @@ void dma_async_device_unregister(struct dma_device *device)
mutex_unlock(&dma_list_mutex);
list_for_each_entry(chan, &device->channels, device_node) {
- dma_clients_notify_removed(chan);
+ WARN_ONCE(chan->client_count,
+ "%s called while %d clients hold a reference\n",
+ __func__, chan->client_count);
device_unregister(&chan->dev);
dma_chan_release(chan);
}
diff --git a/drivers/dma/dmatest.c b/drivers/dma/dmatest.c
index ed9636bfb54..db405088471 100644
--- a/drivers/dma/dmatest.c
+++ b/drivers/dma/dmatest.c
@@ -215,7 +215,6 @@ static int dmatest_func(void *data)
smp_rmb();
chan = thread->chan;
- dma_chan_get(chan);
while (!kthread_should_stop()) {
total_tests++;
@@ -293,7 +292,6 @@ static int dmatest_func(void *data)
}
ret = 0;
- dma_chan_put(chan);
kfree(thread->dstbuf);
err_dstbuf:
kfree(thread->srcbuf);
diff --git a/drivers/dma/dw_dmac.c b/drivers/dma/dw_dmac.c
index 0778d99aea7..377dafa37a2 100644
--- a/drivers/dma/dw_dmac.c
+++ b/drivers/dma/dw_dmac.c
@@ -773,7 +773,7 @@ static int dwc_alloc_chan_resources(struct dma_chan *chan,
dev_vdbg(&chan->dev, "alloc_chan_resources\n");
/* Channels doing slave DMA can only handle one client. */
- if (dwc->dws || client->slave) {
+ if (dwc->dws || (client && client->slave)) {
if (chan->client_count)
return -EBUSY;
}
diff --git a/drivers/mmc/host/atmel-mci.c b/drivers/mmc/host/atmel-mci.c
index 7a3f2436b01..6c11f4d4c4e 100644
--- a/drivers/mmc/host/atmel-mci.c
+++ b/drivers/mmc/host/atmel-mci.c
@@ -593,10 +593,8 @@ atmci_submit_data_dma(struct atmel_mci *host, struct mmc_data *data)
/* If we don't have a channel, we can't do DMA */
chan = host->dma.chan;
- if (chan) {
- dma_chan_get(chan);
+ if (chan)
host->data_chan = chan;
- }
if (!chan)
return -ENODEV;
diff --git a/include/linux/dmaengine.h b/include/linux/dmaengine.h
index e4ec7e7b805..d18d37d1015 100644
--- a/include/linux/dmaengine.h
+++ b/include/linux/dmaengine.h
@@ -165,7 +165,6 @@ struct dma_slave {
*/
struct dma_chan_percpu {
- local_t refcount;
/* stats */
unsigned long memcpy_count;
unsigned long bytes_transferred;
@@ -205,26 +204,6 @@ struct dma_chan {
void dma_chan_cleanup(struct kref *kref);
-static inline void dma_chan_get(struct dma_chan *chan)
-{
- if (unlikely(chan->slow_ref))
- kref_get(&chan->refcount);
- else {
- local_inc(&(per_cpu_ptr(chan->local, get_cpu())->refcount));
- put_cpu();
- }
-}
-
-static inline void dma_chan_put(struct dma_chan *chan)
-{
- if (unlikely(chan->slow_ref))
- kref_put(&chan->refcount, dma_chan_cleanup);
- else {
- local_dec(&(per_cpu_ptr(chan->local, get_cpu())->refcount));
- put_cpu();
- }
-}
-
/*
* typedef dma_event_callback - function pointer to a DMA event callback
* For each channel added to the system this routine is called for each client.
diff --git a/include/net/netdma.h b/include/net/netdma.h
index f28c6e064e8..cbe2737f4a6 100644
--- a/include/net/netdma.h
+++ b/include/net/netdma.h
@@ -27,11 +27,11 @@
static inline struct dma_chan *get_softnet_dma(void)
{
struct dma_chan *chan;
+
rcu_read_lock();
chan = rcu_dereference(__get_cpu_var(softnet_data).net_dma);
- if (chan)
- dma_chan_get(chan);
rcu_read_unlock();
+
return chan;
}
diff --git a/net/ipv4/tcp.c b/net/ipv4/tcp.c
index f28acf11fc6..75e0e0a2d8d 100644
--- a/net/ipv4/tcp.c
+++ b/net/ipv4/tcp.c
@@ -1632,7 +1632,6 @@ skip_copy:
/* Safe to free early-copied skbs now */
__skb_queue_purge(&sk->sk_async_wait_queue);
- dma_chan_put(tp->ucopy.dma_chan);
tp->ucopy.dma_chan = NULL;
}
if (tp->ucopy.pinned_list) {
--
cgit v1.2.3-70-g09d2
From bec085134e446577a983f17f57d642a88d1af53b Mon Sep 17 00:00:00 2001
From: Dan Williams <dan.j.williams@intel.com>
Date: Tue, 6 Jan 2009 11:38:14 -0700
Subject: dmaengine: centralize channel allocation, introduce dma_find_channel
Allowing multiple clients to each define their own channel allocation
scheme quickly leads to a pathological situation. For memory-to-memory
offload all clients can share a central allocator.
This simply moves the existing async_tx allocator to dmaengine with
minimal fixups:
* async_tx.c:get_chan_ref_by_cap --> dmaengine.c:nth_chan
* async_tx.c:async_tx_rebalance --> dmaengine.c:dma_channel_rebalance
* split out common code from async_tx.c:__async_tx_find_channel -->
dma_find_channel
Reviewed-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Dan Williams <dan.j.williams@intel.com>
---
crypto/async_tx/async_tx.c | 146 ++-------------------------------------
drivers/dma/dmaengine.c | 168 ++++++++++++++++++++++++++++++++++++++++++++-
include/linux/dmaengine.h | 3 +
3 files changed, 174 insertions(+), 143 deletions(-)
(limited to 'crypto/async_tx')
diff --git a/crypto/async_tx/async_tx.c b/crypto/async_tx/async_tx.c
index 43fe4cbe71e..b88bb1f608f 100644
--- a/crypto/async_tx/async_tx.c
+++ b/crypto/async_tx/async_tx.c
@@ -37,26 +37,11 @@ static struct dma_client async_tx_dma = {
/* .cap_mask == 0 defaults to all channels */
};
-/**
- * dma_cap_mask_all - enable iteration over all operation types
- */
-static dma_cap_mask_t dma_cap_mask_all;
-
-/**
- * chan_ref_percpu - tracks channel allocations per core/opertion
- */
-struct chan_ref_percpu {
- struct dma_chan_ref *ref;
-};
-
-static int channel_table_initialized;
-static struct chan_ref_percpu *channel_table[DMA_TX_TYPE_END];
-
/**
* async_tx_lock - protect modification of async_tx_master_list and serialize
* rebalance operations
*/
-static spinlock_t async_tx_lock;
+static DEFINE_SPINLOCK(async_tx_lock);
static LIST_HEAD(async_tx_master_list);
@@ -89,85 +74,6 @@ init_dma_chan_ref(struct dma_chan_ref *ref, struct dma_chan *chan)
atomic_set(&ref->count, 0);
}
-/**
- * get_chan_ref_by_cap - returns the nth channel of the given capability
- * defaults to returning the channel with the desired capability and the
- * lowest reference count if the index can not be satisfied
- * @cap: capability to match
- * @index: nth channel desired, passing -1 has the effect of forcing the
- * default return value
- */
-static struct dma_chan_ref *
-get_chan_ref_by_cap(enum dma_transaction_type cap, int index)
-{
- struct dma_chan_ref *ret_ref = NULL, *min_ref = NULL, *ref;
-
- rcu_read_lock();
- list_for_each_entry_rcu(ref, &async_tx_master_list, node)
- if (dma_has_cap(cap, ref->chan->device->cap_mask)) {
- if (!min_ref)
- min_ref = ref;
- else if (atomic_read(&ref->count) <
- atomic_read(&min_ref->count))
- min_ref = ref;
-
- if (index-- == 0) {
- ret_ref = ref;
- break;
- }
- }
- rcu_read_unlock();
-
- if (!ret_ref)
- ret_ref = min_ref;
-
- if (ret_ref)
- atomic_inc(&ret_ref->count);
-
- return ret_ref;
-}
-
-/**
- * async_tx_rebalance - redistribute the available channels, optimize
- * for cpu isolation in the SMP case, and opertaion isolation in the
- * uniprocessor case
- */
-static void async_tx_rebalance(void)
-{
- int cpu, cap, cpu_idx = 0;
- unsigned long flags;
-
- if (!channel_table_initialized)
- return;
-
- spin_lock_irqsave(&async_tx_lock, flags);
-
- /* undo the last distribution */
- for_each_dma_cap_mask(cap, dma_cap_mask_all)
- for_each_possible_cpu(cpu) {
- struct dma_chan_ref *ref =
- per_cpu_ptr(channel_table[cap], cpu)->ref;
- if (ref) {
- atomic_set(&ref->count, 0);
- per_cpu_ptr(channel_table[cap], cpu)->ref =
- NULL;
- }
- }
-
- for_each_dma_cap_mask(cap, dma_cap_mask_all)
- for_each_online_cpu(cpu) {
- struct dma_chan_ref *new;
- if (NR_CPUS > 1)
- new = get_chan_ref_by_cap(cap, cpu_idx++);
- else
- new = get_chan_ref_by_cap(cap, -1);
-
- per_cpu_ptr(channel_table[cap], cpu)->ref = new;
- }
-
- spin_unlock_irqrestore(&async_tx_lock, flags);
-}
-
static enum dma_state_client
dma_channel_add_remove(struct dma_client *client,
struct dma_chan *chan, enum dma_state state)
@@ -211,8 +117,6 @@ dma_channel_add_remove(struct dma_client *client,
" (-ENOMEM)\n");
return 0;
}
-
- async_tx_rebalance();
break;
case DMA_RESOURCE_REMOVED:
found = 0;
@@ -233,8 +137,6 @@ dma_channel_add_remove(struct dma_client *client,
ack = DMA_ACK;
else
break;
-
- async_tx_rebalance();
break;
case DMA_RESOURCE_SUSPEND:
case DMA_RESOURCE_RESUME:
@@ -248,51 +150,18 @@ dma_channel_add_remove(struct dma_client *client,
return ack;
}
-static int __init
-async_tx_init(void)
+static int __init async_tx_init(void)
{
- enum dma_transaction_type cap;
-
- spin_lock_init(&async_tx_lock);
- bitmap_fill(dma_cap_mask_all.bits, DMA_TX_TYPE_END);
-
- /* an interrupt will never be an explicit operation type.
- * clearing this bit prevents allocation to a slot in 'channel_table'
- */
- clear_bit(DMA_INTERRUPT, dma_cap_mask_all.bits);
-
- for_each_dma_cap_mask(cap, dma_cap_mask_all) {
- channel_table[cap] = alloc_percpu(struct chan_ref_percpu);
- if (!channel_table[cap])
- goto err;
- }
-
- channel_table_initialized = 1;
dma_async_client_register(&async_tx_dma);
dma_async_client_chan_request(&async_tx_dma);
printk(KERN_INFO "async_tx: api initialized (async)\n");
return 0;
-err:
- printk(KERN_ERR "async_tx: initialization failure\n");
-
- while (--cap >= 0)
- free_percpu(channel_table[cap]);
-
- return 1;
}
static void __exit async_tx_exit(void)
{
- enum dma_transaction_type cap;
-
- channel_table_initialized = 0;
-
- for_each_dma_cap_mask(cap, dma_cap_mask_all)
- if (channel_table[cap])
- free_percpu(channel_table[cap]);
-
dma_async_client_unregister(&async_tx_dma);
}
@@ -308,16 +177,9 @@ __async_tx_find_channel(struct dma_async_tx_descriptor *depend_tx,
{
/* see if we can keep the chain on one channel */
if (depend_tx &&
- dma_has_cap(tx_type, depend_tx->chan->device->cap_mask))
+ dma_has_cap(tx_type, depend_tx->chan->device->cap_mask))
return depend_tx->chan;
- else if (likely(channel_table_initialized)) {
- struct dma_chan_ref *ref;
- int cpu = get_cpu();
- ref = per_cpu_ptr(channel_table[tx_type], cpu)->ref;
- put_cpu();
- return ref ? ref->chan : NULL;
- } else
- return NULL;
+ return dma_find_channel(tx_type);
}
EXPORT_SYMBOL_GPL(__async_tx_find_channel);
#else
diff --git a/drivers/dma/dmaengine.c b/drivers/dma/dmaengine.c
index d4d925912c4..87a8cd4791e 100644
--- a/drivers/dma/dmaengine.c
+++ b/drivers/dma/dmaengine.c
@@ -294,6 +294,164 @@ static void dma_chan_release(struct dma_chan *chan)
call_rcu(&chan->rcu, dma_chan_free_rcu);
}
+/**
+ * dma_cap_mask_all - enable iteration over all operation types
+ */
+static dma_cap_mask_t dma_cap_mask_all;
+
+/**
+ * dma_chan_tbl_ent - tracks channel allocations per core/operation
+ * @chan - associated channel for this entry
+ */
+struct dma_chan_tbl_ent {
+ struct dma_chan *chan;
+};
+
+/**
+ * channel_table - percpu lookup table for memory-to-memory offload providers
+ */
+static struct dma_chan_tbl_ent *channel_table[DMA_TX_TYPE_END];
+
+static int __init dma_channel_table_init(void)
+{
+ enum dma_transaction_type cap;
+ int err = 0;
+
+ bitmap_fill(dma_cap_mask_all.bits, DMA_TX_TYPE_END);
+
+ /* 'interrupt' and 'slave' are channel capabilities, but are not
+ * associated with an operation so they do not need an entry in the
+ * channel_table
+ */
+ clear_bit(DMA_INTERRUPT, dma_cap_mask_all.bits);
+ clear_bit(DMA_SLAVE, dma_cap_mask_all.bits);
+
+ for_each_dma_cap_mask(cap, dma_cap_mask_all) {
+ channel_table[cap] = alloc_percpu(struct dma_chan_tbl_ent);
+ if (!channel_table[cap]) {
+ err = -ENOMEM;
+ break;
+ }
+ }
+
+ if (err) {
+ pr_err("dmaengine: initialization failure\n");
+ for_each_dma_cap_mask(cap, dma_cap_mask_all)
+ if (channel_table[cap])
+ free_percpu(channel_table[cap]);
+ }
+
+ return err;
+}
+subsys_initcall(dma_channel_table_init);
+
+/**
+ * dma_find_channel - find a channel to carry out the operation
+ * @tx_type: transaction type
+ */
+struct dma_chan *dma_find_channel(enum dma_transaction_type tx_type)
+{
+ struct dma_chan *chan;
+ int cpu;
+
+ WARN_ONCE(dmaengine_ref_count == 0,
+ "client called %s without a reference", __func__);
+
+ cpu = get_cpu();
+ chan = per_cpu_ptr(channel_table[tx_type], cpu)->chan;
+ put_cpu();
+
+ return chan;
+}
+EXPORT_SYMBOL(dma_find_channel);
+
+/**
+ * nth_chan - returns the nth channel of the given capability
+ * @cap: capability to match
+ * @n: nth channel desired
+ *
+ * Defaults to returning the channel with the desired capability and the
+ * lowest reference count when 'n' cannot be satisfied. Must be called
+ * under dma_list_mutex.
+ */
+static struct dma_chan *nth_chan(enum dma_transaction_type cap, int n)
+{
+ struct dma_device *device;
+ struct dma_chan *chan;
+ struct dma_chan *ret = NULL;
+ struct dma_chan *min = NULL;
+
+ list_for_each_entry(device, &dma_device_list, global_node) {
+ if (!dma_has_cap(cap, device->cap_mask))
+ continue;
+ list_for_each_entry(chan, &device->channels, device_node) {
+ if (!chan->client_count)
+ continue;
+ if (!min)
+ min = chan;
+ else if (chan->table_count < min->table_count)
+ min = chan;
+
+ if (n-- == 0) {
+ ret = chan;
+ break; /* done */
+ }
+ }
+ if (ret)
+ break; /* done */
+ }
+
+ if (!ret)
+ ret = min;
+
+ if (ret)
+ ret->table_count++;
+
+ return ret;
+}
+
+/**
+ * dma_channel_rebalance - redistribute the available channels
+ *
+ * Optimize for cpu isolation (each cpu gets a dedicated channel for an
+ * operation type) in the SMP case, and operation isolation (avoid
+ * multi-tasking channels) in the non-SMP case. Must be called under
+ * dma_list_mutex.
+ */
+static void dma_channel_rebalance(void)
+{
+ struct dma_chan *chan;
+ struct dma_device *device;
+ int cpu;
+ int cap;
+ int n;
+
+ /* undo the last distribution */
+ for_each_dma_cap_mask(cap, dma_cap_mask_all)
+ for_each_possible_cpu(cpu)
+ per_cpu_ptr(channel_table[cap], cpu)->chan = NULL;
+
+ list_for_each_entry(device, &dma_device_list, global_node)
+ list_for_each_entry(chan, &device->channels, device_node)
+ chan->table_count = 0;
+
+ /* don't populate the channel_table if no clients are available */
+ if (!dmaengine_ref_count)
+ return;
+
+ /* redistribute available channels */
+ n = 0;
+ for_each_dma_cap_mask(cap, dma_cap_mask_all)
+ for_each_online_cpu(cpu) {
+ if (num_possible_cpus() > 1)
+ chan = nth_chan(cap, n++);
+ else
+ chan = nth_chan(cap, -1);
+
+ per_cpu_ptr(channel_table[cap], cpu)->chan = chan;
+ }
+}
+
/**
* dma_chans_notify_available - broadcast available channels to the clients
*/
@@ -339,7 +497,12 @@ void dma_async_client_register(struct dma_client *client)
dev_name(&chan->dev), err);
}
-
+ /* if this is the first reference and there were channels
+ * waiting we need to rebalance to get those channels
+ * incorporated into the channel table
+ */
+ if (dmaengine_ref_count == 1)
+ dma_channel_rebalance();
list_add_tail(&client->global_node, &dma_client_list);
mutex_unlock(&dma_list_mutex);
}
@@ -473,6 +636,7 @@ int dma_async_device_register(struct dma_device *device)
}
}
list_add_tail(&device->global_node, &dma_device_list);
+ dma_channel_rebalance();
mutex_unlock(&dma_list_mutex);
dma_clients_notify_available();
@@ -514,6 +678,7 @@ void dma_async_device_unregister(struct dma_device *device)
mutex_lock(&dma_list_mutex);
list_del(&device->global_node);
+ dma_channel_rebalance();
mutex_unlock(&dma_list_mutex);
list_for_each_entry(chan, &device->channels, device_node) {
@@ -768,3 +933,4 @@ static int __init dma_bus_init(void)
}
subsys_initcall(dma_bus_init);
+
diff --git a/include/linux/dmaengine.h b/include/linux/dmaengine.h
index d18d37d1015..b466f02e243 100644
--- a/include/linux/dmaengine.h
+++ b/include/linux/dmaengine.h
@@ -182,6 +182,7 @@ struct dma_chan_percpu {
* @device_node: used to add this to the device chan list
* @local: per-cpu pointer to a struct dma_chan_percpu
* @client-count: how many clients are using this channel
+ * @table_count: number of appearances in the mem-to-mem allocation table
*/
struct dma_chan {
struct dma_device *device;
@@ -198,6 +199,7 @@ struct dma_chan {
struct list_head device_node;
struct dma_chan_percpu *local;
int client_count;
+ int table_count;
};
#define to_dma_chan(p) container_of(p, struct dma_chan, dev)
@@ -468,6 +470,7 @@ static inline enum dma_status dma_wait_for_async_tx(struct dma_async_tx_descript
int dma_async_device_register(struct dma_device *device);
void dma_async_device_unregister(struct dma_device *device);
void dma_run_dependencies(struct dma_async_tx_descriptor *tx);
+struct dma_chan *dma_find_channel(enum dma_transaction_type tx_type);
/* --- Helper iov-locking functions --- */
--
cgit v1.2.3-70-g09d2
From 2ba05622b8b143b0c95968ba59bddfbd6d2f2559 Mon Sep 17 00:00:00 2001
From: Dan Williams <dan.j.williams@intel.com>
Date: Tue, 6 Jan 2009 11:38:14 -0700
Subject: dmaengine: provide a common 'issue_pending_all' implementation
async_tx and net_dma each have open-coded versions of issue_pending_all,
so provide a common routine in dmaengine.
The implementation needs to walk the global device list, so implement
rcu to allow dma_issue_pending_all to run lockless. Clients protect
themselves from channel removal events by holding a dmaengine reference.
Reviewed-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Dan Williams <dan.j.williams@intel.com>
---
crypto/async_tx/async_tx.c | 12 ------------
drivers/dma/dmaengine.c | 27 ++++++++++++++++++++++++---
include/linux/async_tx.h | 2 +-
include/linux/dmaengine.h | 1 +
net/core/dev.c | 9 +--------
5 files changed, 27 insertions(+), 24 deletions(-)
(limited to 'crypto/async_tx')
diff --git a/crypto/async_tx/async_tx.c b/crypto/async_tx/async_tx.c
index b88bb1f608f..2cdf7a0867b 100644
--- a/crypto/async_tx/async_tx.c
+++ b/crypto/async_tx/async_tx.c
@@ -45,18 +45,6 @@ static DEFINE_SPINLOCK(async_tx_lock);
static LIST_HEAD(async_tx_master_list);
-/* async_tx_issue_pending_all - start all transactions on all channels */
-void async_tx_issue_pending_all(void)
-{
- struct dma_chan_ref *ref;
-
- rcu_read_lock();
- list_for_each_entry_rcu(ref, &async_tx_master_list, node)
- ref->chan->device->device_issue_pending(ref->chan);
- rcu_read_unlock();
-}
-EXPORT_SYMBOL_GPL(async_tx_issue_pending_all);
-
static void
free_dma_chan_ref(struct rcu_head *rcu)
{
diff --git a/drivers/dma/dmaengine.c b/drivers/dma/dmaengine.c
index 87a8cd4791e..418eca28d47 100644
--- a/drivers/dma/dmaengine.c
+++ b/drivers/dma/dmaengine.c
@@ -70,6 +70,7 @@
#include <linux/rcupdate.h>
#include <linux/mutex.h>
#include <linux/jiffies.h>
+#include <linux/rculist.h>
static DEFINE_MUTEX(dma_list_mutex);
static LIST_HEAD(dma_device_list);
@@ -365,6 +366,26 @@ struct dma_chan *dma_find_channel(enum dma_transaction_type tx_type)
}
EXPORT_SYMBOL(dma_find_channel);
+/**
+ * dma_issue_pending_all - flush all pending operations across all channels
+ */
+void dma_issue_pending_all(void)
+{
+ struct dma_device *device;
+ struct dma_chan *chan;
+
+ WARN_ONCE(dmaengine_ref_count == 0,
+ "client called %s without a reference", __func__);
+
+ rcu_read_lock();
+ list_for_each_entry_rcu(device, &dma_device_list, global_node)
+ list_for_each_entry(chan, &device->channels, device_node)
+ if (chan->client_count)
+ device->device_issue_pending(chan);
+ rcu_read_unlock();
+}
+EXPORT_SYMBOL(dma_issue_pending_all);
+
/**
* nth_chan - returns the nth channel of the given capability
* @cap: capability to match
@@ -490,7 +511,7 @@ void dma_async_client_register(struct dma_client *client)
err = dma_chan_get(chan);
if (err == -ENODEV) {
/* module removed before we could use it */
- list_del_init(&device->global_node);
+ list_del_rcu(&device->global_node);
break;
} else if (err)
pr_err("dmaengine: failed to get %s: (%d)\n",
@@ -635,7 +656,7 @@ int dma_async_device_register(struct dma_device *device)
goto err_out;
}
}
- list_add_tail(&device->global_node, &dma_device_list);
+ list_add_tail_rcu(&device->global_node, &dma_device_list);
dma_channel_rebalance();
mutex_unlock(&dma_list_mutex);
@@ -677,7 +698,7 @@ void dma_async_device_unregister(struct dma_device *device)
struct dma_chan *chan;
mutex_lock(&dma_list_mutex);
- list_del(&device->global_node);
+ list_del_rcu(&device->global_node);
dma_channel_rebalance();
mutex_unlock(&dma_list_mutex);
diff --git a/include/linux/async_tx.h b/include/linux/async_tx.h
index 1c816775f13..45f6297821b 100644
--- a/include/linux/async_tx.h
+++ b/include/linux/async_tx.h
@@ -59,7 +59,7 @@ enum async_tx_flags {
};
#ifdef CONFIG_DMA_ENGINE
-void async_tx_issue_pending_all(void);
+#define async_tx_issue_pending_all dma_issue_pending_all
#ifdef CONFIG_ARCH_HAS_ASYNC_TX_FIND_CHANNEL
#include <asm/async_tx.h>
#else
diff --git a/include/linux/dmaengine.h b/include/linux/dmaengine.h
index b466f02e243..57a43adfc39 100644
--- a/include/linux/dmaengine.h
+++ b/include/linux/dmaengine.h
@@ -471,6 +471,7 @@ int dma_async_device_register(struct dma_device *device);
void dma_async_device_unregister(struct dma_device *device);
void dma_run_dependencies(struct dma_async_tx_descriptor *tx);
struct dma_chan *dma_find_channel(enum dma_transaction_type tx_type);
+void dma_issue_pending_all(void);
/* --- Helper iov-locking functions --- */
diff --git a/net/core/dev.c b/net/core/dev.c
index 09c66a449da..e40b0d57f8f 100644
--- a/net/core/dev.c
+++ b/net/core/dev.c
@@ -2635,14 +2635,7 @@ out:
* There may not be any more sk_buffs coming right now, so push
* any pending DMA copies to hardware
*/
- if (!cpus_empty(net_dma.channel_mask)) {
- int chan_idx;
- for_each_cpu_mask_nr(chan_idx, net_dma.channel_mask) {
- struct dma_chan *chan = net_dma.channels[chan_idx];
- if (chan)
- dma_async_memcpy_issue_pending(chan);
- }
- }
+ dma_issue_pending_all();
#endif
return;
--
cgit v1.2.3-70-g09d2
From 209b84a88fe81341b4d8d465acc4a67cb7c3feb3 Mon Sep 17 00:00:00 2001
From: Dan Williams <dan.j.williams@intel.com>
Date: Tue, 6 Jan 2009 11:38:17 -0700
Subject: dmaengine: replace dma_async_client_register with dmaengine_get
Now that clients no longer need to be notified of channel arrival
dma_async_client_register can simply increment the dmaengine_ref_count.
Reviewed-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Dan Williams <dan.j.williams@intel.com>
---
crypto/async_tx/async_tx.c | 115 +--------------------------------------------
drivers/dma/dmaengine.c | 22 +++------
include/linux/dmaengine.h | 4 +-
net/core/dev.c | 3 +-
4 files changed, 11 insertions(+), 133 deletions(-)
(limited to 'crypto/async_tx')
diff --git a/crypto/async_tx/async_tx.c b/crypto/async_tx/async_tx.c
index 2cdf7a0867b..f21147f3626 100644
--- a/crypto/async_tx/async_tx.c
+++ b/crypto/async_tx/async_tx.c
@@ -28,120 +28,9 @@
#include <linux/async_tx.h>
#ifdef CONFIG_DMA_ENGINE
-static enum dma_state_client
-dma_channel_add_remove(struct dma_client *client,
- struct dma_chan *chan, enum dma_state state);
-
-static struct dma_client async_tx_dma = {
- .event_callback = dma_channel_add_remove,
- /* .cap_mask == 0 defaults to all channels */
-};
-
-/**
- * async_tx_lock - protect modification of async_tx_master_list and serialize
- * rebalance operations
- */
-static DEFINE_SPINLOCK(async_tx_lock);
-
-static LIST_HEAD(async_tx_master_list);
-
-static void
-free_dma_chan_ref(struct rcu_head *rcu)
-{
- struct dma_chan_ref *ref;
- ref = container_of(rcu, struct dma_chan_ref, rcu);
- kfree(ref);
-}
-
-static void
-init_dma_chan_ref(struct dma_chan_ref *ref, struct dma_chan *chan)
-{
- INIT_LIST_HEAD(&ref->node);
- INIT_RCU_HEAD(&ref->rcu);
- ref->chan = chan;
- atomic_set(&ref->count, 0);
-}
-
-static enum dma_state_client
-dma_channel_add_remove(struct dma_client *client,
- struct dma_chan *chan, enum dma_state state)
-{
- unsigned long found, flags;
- struct dma_chan_ref *master_ref, *ref;
- enum dma_state_client ack = DMA_DUP; /* default: take no action */
-
- switch (state) {
- case DMA_RESOURCE_AVAILABLE:
- found = 0;
- rcu_read_lock();
- list_for_each_entry_rcu(ref, &async_tx_master_list, node)
- if (ref->chan == chan) {
- found = 1;
- break;
- }
- rcu_read_unlock();
-
- pr_debug("async_tx: dma resource available [%s]\n",
- found ? "old" : "new");
-
- if (!found)
- ack = DMA_ACK;
- else
- break;
-
- /* add the channel to the generic management list */
- master_ref = kmalloc(sizeof(*master_ref), GFP_KERNEL);
- if (master_ref) {
- init_dma_chan_ref(master_ref, chan);
- spin_lock_irqsave(&async_tx_lock, flags);
- list_add_tail_rcu(&master_ref->node,
- &async_tx_master_list);
- spin_unlock_irqrestore(&async_tx_lock,
- flags);
- } else {
- printk(KERN_WARNING "async_tx: unable to create"
- " new master entry in response to"
- " a DMA_RESOURCE_ADDED event"
- " (-ENOMEM)\n");
- return 0;
- }
- break;
- case DMA_RESOURCE_REMOVED:
- found = 0;
- spin_lock_irqsave(&async_tx_lock, flags);
- list_for_each_entry(ref, &async_tx_master_list, node)
- if (ref->chan == chan) {
- list_del_rcu(&ref->node);
- call_rcu(&ref->rcu, free_dma_chan_ref);
- found = 1;
- break;
- }
- spin_unlock_irqrestore(&async_tx_lock, flags);
-
- pr_debug("async_tx: dma resource removed [%s]\n",
- found ? "ours" : "not ours");
-
- if (found)
- ack = DMA_ACK;
- else
- break;
- break;
- case DMA_RESOURCE_SUSPEND:
- case DMA_RESOURCE_RESUME:
- printk(KERN_WARNING "async_tx: does not support dma channel"
- " suspend/resume\n");
- break;
- default:
- BUG();
- }
-
- return ack;
-}
-
static int __init async_tx_init(void)
{
- dma_async_client_register(&async_tx_dma);
- dma_async_client_chan_request(&async_tx_dma);
+ dmaengine_get();
printk(KERN_INFO "async_tx: api initialized (async)\n");
@@ -150,7 +39,7 @@ static int __init async_tx_init(void)
static void __exit async_tx_exit(void)
{
- dma_async_client_unregister(&async_tx_dma);
+ dmaengine_put();
}
/**
diff --git a/drivers/dma/dmaengine.c b/drivers/dma/dmaengine.c
index 90aca505a1d..3f1849b7f5e 100644
--- a/drivers/dma/dmaengine.c
+++ b/drivers/dma/dmaengine.c
@@ -600,10 +600,9 @@ static void dma_clients_notify_available(void)
}
/**
- * dma_async_client_register - register a &dma_client
- * @client: ptr to a client structure with valid 'event_callback' and 'cap_mask'
+ * dmaengine_get - register interest in dma_channels
*/
-void dma_async_client_register(struct dma_client *client)
+void dmaengine_get(void)
{
struct dma_device *device, *_d;
struct dma_chan *chan;
@@ -634,25 +633,18 @@ void dma_async_client_register(struct dma_client *client)
*/
if (dmaengine_ref_count == 1)
dma_channel_rebalance();
- list_add_tail(&client->global_node, &dma_client_list);
mutex_unlock(&dma_list_mutex);
}
-EXPORT_SYMBOL(dma_async_client_register);
+EXPORT_SYMBOL(dmaengine_get);
/**
- * dma_async_client_unregister - unregister a client and free the &dma_client
- * @client: &dma_client to free
- *
- * Force frees any allocated DMA channels, frees the &dma_client memory
+ * dmaengine_put - let dma drivers be removed when ref_count == 0
*/
-void dma_async_client_unregister(struct dma_client *client)
+void dmaengine_put(void)
{
struct dma_device *device;
struct dma_chan *chan;
- if (!client)
- return;
-
mutex_lock(&dma_list_mutex);
dmaengine_ref_count--;
BUG_ON(dmaengine_ref_count < 0);
@@ -663,11 +655,9 @@ void dma_async_client_unregister(struct dma_client *client)
list_for_each_entry(chan, &device->channels, device_node)
dma_chan_put(chan);
}
-
- list_del(&client->global_node);
mutex_unlock(&dma_list_mutex);
}
-EXPORT_SYMBOL(dma_async_client_unregister);
+EXPORT_SYMBOL(dmaengine_put);
/**
* dma_async_client_chan_request - send all available channels to the
diff --git a/include/linux/dmaengine.h b/include/linux/dmaengine.h
index d63544cf8a1..37d95db156d 100644
--- a/include/linux/dmaengine.h
+++ b/include/linux/dmaengine.h
@@ -318,8 +318,8 @@ struct dma_device {
/* --- public DMA engine API --- */
-void dma_async_client_register(struct dma_client *client);
-void dma_async_client_unregister(struct dma_client *client);
+void dmaengine_get(void);
+void dmaengine_put(void);
void dma_async_client_chan_request(struct dma_client *client);
dma_cookie_t dma_async_memcpy_buf_to_buf(struct dma_chan *chan,
void *dest, void *src, size_t len);
diff --git a/net/core/dev.c b/net/core/dev.c
index bbb07dbe174..7596fc9403c 100644
--- a/net/core/dev.c
+++ b/net/core/dev.c
@@ -4894,8 +4894,7 @@ static int __init netdev_dma_register(void)
}
spin_lock_init(&net_dma.lock);
dma_cap_set(DMA_MEMCPY, net_dma.client.cap_mask);
- dma_async_client_register(&net_dma.client);
- dma_async_client_chan_request(&net_dma.client);
+ dmaengine_get();
return 0;
}
--
cgit v1.2.3-70-g09d2