diff options
Diffstat (limited to 'drivers/net/sfc/tx.c')
| -rw-r--r-- | drivers/net/sfc/tx.c | 116 |
1 files changed, 51 insertions, 65 deletions
diff --git a/drivers/net/sfc/tx.c b/drivers/net/sfc/tx.c index 489c4de3144..c54fa30e627 100644 --- a/drivers/net/sfc/tx.c +++ b/drivers/net/sfc/tx.c @@ -15,7 +15,6 @@ #include <linux/if_ether.h> #include <linux/highmem.h> #include "net_driver.h" -#include "tx.h" #include "efx.h" #include "falcon.h" #include "workarounds.h" @@ -26,8 +25,7 @@ * The tx_queue descriptor ring fill-level must fall below this value * before we restart the netif queue */ -#define EFX_NETDEV_TX_THRESHOLD(_tx_queue) \ - (_tx_queue->efx->type->txd_ring_mask / 2u) +#define EFX_TXQ_THRESHOLD (EFX_TXQ_MASK / 2u) /* We want to be able to nest calls to netif_stop_queue(), since each * channel can have an individual stop on the queue. @@ -125,6 +123,24 @@ static void efx_tsoh_free(struct efx_tx_queue *tx_queue, } +static inline unsigned +efx_max_tx_len(struct efx_nic *efx, dma_addr_t dma_addr) +{ + /* Depending on the NIC revision, we can use descriptor + * lengths up to 8K or 8K-1. However, since PCI Express + * devices must split read requests at 4K boundaries, there is + * little benefit from using descriptors that cross those + * boundaries and we keep things simple by not doing so. + */ + unsigned len = (~dma_addr & 0xfff) + 1; + + /* Work around hardware bug for unaligned buffers. */ + if (EFX_WORKAROUND_5391(efx) && (dma_addr & 0xf)) + len = min_t(unsigned, len, 512 - (dma_addr & 0xf)); + + return len; +} + /* * Add a socket buffer to a TX queue * @@ -135,11 +151,13 @@ static void efx_tsoh_free(struct efx_tx_queue *tx_queue, * If any DMA mapping fails, any mapped fragments will be unmapped, * the queue's insert pointer will be restored to its original value. * + * This function is split out from efx_hard_start_xmit to allow the + * loopback test to direct packets via specific TX queues. + * * Returns NETDEV_TX_OK or NETDEV_TX_BUSY * You must hold netif_tx_lock() to call this function. */ -static netdev_tx_t efx_enqueue_skb(struct efx_tx_queue *tx_queue, - struct sk_buff *skb) +netdev_tx_t efx_enqueue_skb(struct efx_tx_queue *tx_queue, struct sk_buff *skb) { struct efx_nic *efx = tx_queue->efx; struct pci_dev *pci_dev = efx->pci_dev; @@ -147,7 +165,7 @@ static netdev_tx_t efx_enqueue_skb(struct efx_tx_queue *tx_queue, skb_frag_t *fragment; struct page *page; int page_offset; - unsigned int len, unmap_len = 0, fill_level, insert_ptr, misalign; + unsigned int len, unmap_len = 0, fill_level, insert_ptr; dma_addr_t dma_addr, unmap_addr = 0; unsigned int dma_len; bool unmap_single; @@ -156,7 +174,7 @@ static netdev_tx_t efx_enqueue_skb(struct efx_tx_queue *tx_queue, EFX_BUG_ON_PARANOID(tx_queue->write_count != tx_queue->insert_count); - if (skb_shinfo((struct sk_buff *)skb)->gso_size) + if (skb_shinfo(skb)->gso_size) return efx_enqueue_skb_tso(tx_queue, skb); /* Get size of the initial fragment */ @@ -171,7 +189,7 @@ static netdev_tx_t efx_enqueue_skb(struct efx_tx_queue *tx_queue, } fill_level = tx_queue->insert_count - tx_queue->old_read_count; - q_space = efx->type->txd_ring_mask - 1 - fill_level; + q_space = EFX_TXQ_MASK - 1 - fill_level; /* Map for DMA. Use pci_map_single rather than pci_map_page * since this is more efficient on machines with sparse @@ -208,16 +226,14 @@ static netdev_tx_t efx_enqueue_skb(struct efx_tx_queue *tx_queue, &tx_queue->read_count; fill_level = (tx_queue->insert_count - tx_queue->old_read_count); - q_space = (efx->type->txd_ring_mask - 1 - - fill_level); + q_space = EFX_TXQ_MASK - 1 - fill_level; if (unlikely(q_space-- <= 0)) goto stop; smp_mb(); --tx_queue->stopped; } - insert_ptr = (tx_queue->insert_count & - efx->type->txd_ring_mask); + insert_ptr = tx_queue->insert_count & EFX_TXQ_MASK; buffer = &tx_queue->buffer[insert_ptr]; efx_tsoh_free(tx_queue, buffer); EFX_BUG_ON_PARANOID(buffer->tsoh); @@ -226,14 +242,10 @@ static netdev_tx_t efx_enqueue_skb(struct efx_tx_queue *tx_queue, EFX_BUG_ON_PARANOID(!buffer->continuation); EFX_BUG_ON_PARANOID(buffer->unmap_len); - dma_len = (((~dma_addr) & efx->type->tx_dma_mask) + 1); - if (likely(dma_len > len)) + dma_len = efx_max_tx_len(efx, dma_addr); + if (likely(dma_len >= len)) dma_len = len; - misalign = (unsigned)dma_addr & efx->type->bug5391_mask; - if (misalign && dma_len + misalign > 512) - dma_len = 512 - misalign; - /* Fill out per descriptor fields */ buffer->len = dma_len; buffer->dma_addr = dma_addr; @@ -276,7 +288,7 @@ static netdev_tx_t efx_enqueue_skb(struct efx_tx_queue *tx_queue, skb_shinfo(skb)->nr_frags + 1); /* Mark the packet as transmitted, and free the SKB ourselves */ - dev_kfree_skb_any((struct sk_buff *)skb); + dev_kfree_skb_any(skb); goto unwind; stop: @@ -289,7 +301,7 @@ static netdev_tx_t efx_enqueue_skb(struct efx_tx_queue *tx_queue, /* Work backwards until we hit the original insert pointer value */ while (tx_queue->insert_count != tx_queue->write_count) { --tx_queue->insert_count; - insert_ptr = tx_queue->insert_count & efx->type->txd_ring_mask; + insert_ptr = tx_queue->insert_count & EFX_TXQ_MASK; buffer = &tx_queue->buffer[insert_ptr]; efx_dequeue_buffer(tx_queue, buffer); buffer->len = 0; @@ -318,10 +330,9 @@ static void efx_dequeue_buffers(struct efx_tx_queue *tx_queue, { struct efx_nic *efx = tx_queue->efx; unsigned int stop_index, read_ptr; - unsigned int mask = tx_queue->efx->type->txd_ring_mask; - stop_index = (index + 1) & mask; - read_ptr = tx_queue->read_count & mask; + stop_index = (index + 1) & EFX_TXQ_MASK; + read_ptr = tx_queue->read_count & EFX_TXQ_MASK; while (read_ptr != stop_index) { struct efx_tx_buffer *buffer = &tx_queue->buffer[read_ptr]; @@ -338,28 +349,10 @@ static void efx_dequeue_buffers(struct efx_tx_queue *tx_queue, buffer->len = 0; ++tx_queue->read_count; - read_ptr = tx_queue->read_count & mask; + read_ptr = tx_queue->read_count & EFX_TXQ_MASK; } } -/* Initiate a packet transmission on the specified TX queue. - * Note that returning anything other than NETDEV_TX_OK will cause the - * OS to free the skb. - * - * This function is split out from efx_hard_start_xmit to allow the - * loopback test to direct packets via specific TX queues. It is - * therefore a non-static inline, so as not to penalise performance - * for non-loopback transmissions. - * - * Context: netif_tx_lock held - */ -inline netdev_tx_t efx_xmit(struct efx_nic *efx, - struct efx_tx_queue *tx_queue, struct sk_buff *skb) -{ - /* Map fragments for DMA and add to TX queue */ - return efx_enqueue_skb(tx_queue, skb); -} - /* Initiate a packet transmission. We use one channel per CPU * (sharing when we have more CPUs than channels). On Falcon, the TX * completion events will be directed back to the CPU that transmitted @@ -383,7 +376,7 @@ netdev_tx_t efx_hard_start_xmit(struct sk_buff *skb, else tx_queue = &efx->tx_queue[EFX_TX_QUEUE_NO_CSUM]; - return efx_xmit(efx, tx_queue, skb); + return efx_enqueue_skb(tx_queue, skb); } void efx_xmit_done(struct efx_tx_queue *tx_queue, unsigned int index) @@ -391,7 +384,7 @@ void efx_xmit_done(struct efx_tx_queue *tx_queue, unsigned int index) unsigned fill_level; struct efx_nic *efx = tx_queue->efx; - EFX_BUG_ON_PARANOID(index > efx->type->txd_ring_mask); + EFX_BUG_ON_PARANOID(index > EFX_TXQ_MASK); efx_dequeue_buffers(tx_queue, index); @@ -401,7 +394,7 @@ void efx_xmit_done(struct efx_tx_queue *tx_queue, unsigned int index) smp_mb(); if (unlikely(tx_queue->stopped) && likely(efx->port_enabled)) { fill_level = tx_queue->insert_count - tx_queue->read_count; - if (fill_level < EFX_NETDEV_TX_THRESHOLD(tx_queue)) { + if (fill_level < EFX_TXQ_THRESHOLD) { EFX_BUG_ON_PARANOID(!efx_dev_registered(efx)); /* Do this under netif_tx_lock(), to avoid racing @@ -425,11 +418,11 @@ int efx_probe_tx_queue(struct efx_tx_queue *tx_queue) EFX_LOG(efx, "creating TX queue %d\n", tx_queue->queue); /* Allocate software ring */ - txq_size = (efx->type->txd_ring_mask + 1) * sizeof(*tx_queue->buffer); + txq_size = EFX_TXQ_SIZE * sizeof(*tx_queue->buffer); tx_queue->buffer = kzalloc(txq_size, GFP_KERNEL); if (!tx_queue->buffer) return -ENOMEM; - for (i = 0; i <= efx->type->txd_ring_mask; ++i) + for (i = 0; i <= EFX_TXQ_MASK; ++i) tx_queue->buffer[i].continuation = true; /* Allocate hardware ring */ @@ -468,8 +461,7 @@ void efx_release_tx_buffers(struct efx_tx_queue *tx_queue) /* Free any buffers left in the ring */ while (tx_queue->read_count != tx_queue->write_count) { - buffer = &tx_queue->buffer[tx_queue->read_count & - tx_queue->efx->type->txd_ring_mask]; + buffer = &tx_queue->buffer[tx_queue->read_count & EFX_TXQ_MASK]; efx_dequeue_buffer(tx_queue, buffer); buffer->continuation = true; buffer->len = 0; @@ -708,14 +700,14 @@ static int efx_tx_queue_insert(struct efx_tx_queue *tx_queue, { struct efx_tx_buffer *buffer; struct efx_nic *efx = tx_queue->efx; - unsigned dma_len, fill_level, insert_ptr, misalign; + unsigned dma_len, fill_level, insert_ptr; int q_space; EFX_BUG_ON_PARANOID(len <= 0); fill_level = tx_queue->insert_count - tx_queue->old_read_count; /* -1 as there is no way to represent all descriptors used */ - q_space = efx->type->txd_ring_mask - 1 - fill_level; + q_space = EFX_TXQ_MASK - 1 - fill_level; while (1) { if (unlikely(q_space-- <= 0)) { @@ -731,7 +723,7 @@ static int efx_tx_queue_insert(struct efx_tx_queue *tx_queue, *(volatile unsigned *)&tx_queue->read_count; fill_level = (tx_queue->insert_count - tx_queue->old_read_count); - q_space = efx->type->txd_ring_mask - 1 - fill_level; + q_space = EFX_TXQ_MASK - 1 - fill_level; if (unlikely(q_space-- <= 0)) { *final_buffer = NULL; return 1; @@ -740,13 +732,13 @@ static int efx_tx_queue_insert(struct efx_tx_queue *tx_queue, --tx_queue->stopped; } - insert_ptr = tx_queue->insert_count & efx->type->txd_ring_mask; + insert_ptr = tx_queue->insert_count & EFX_TXQ_MASK; buffer = &tx_queue->buffer[insert_ptr]; ++tx_queue->insert_count; EFX_BUG_ON_PARANOID(tx_queue->insert_count - tx_queue->read_count > - efx->type->txd_ring_mask); + EFX_TXQ_MASK); efx_tsoh_free(tx_queue, buffer); EFX_BUG_ON_PARANOID(buffer->len); @@ -757,12 +749,7 @@ static int efx_tx_queue_insert(struct efx_tx_queue *tx_queue, buffer->dma_addr = dma_addr; - /* Ensure we do not cross a boundary unsupported by H/W */ - dma_len = (~dma_addr & efx->type->tx_dma_mask) + 1; - - misalign = (unsigned)dma_addr & efx->type->bug5391_mask; - if (misalign && dma_len + misalign > 512) - dma_len = 512 - misalign; + dma_len = efx_max_tx_len(efx, dma_addr); /* If there is enough space to send then do so */ if (dma_len >= len) @@ -792,8 +779,7 @@ static void efx_tso_put_header(struct efx_tx_queue *tx_queue, { struct efx_tx_buffer *buffer; - buffer = &tx_queue->buffer[tx_queue->insert_count & - tx_queue->efx->type->txd_ring_mask]; + buffer = &tx_queue->buffer[tx_queue->insert_count & EFX_TXQ_MASK]; efx_tsoh_free(tx_queue, buffer); EFX_BUG_ON_PARANOID(buffer->len); EFX_BUG_ON_PARANOID(buffer->unmap_len); @@ -818,7 +804,7 @@ static void efx_enqueue_unwind(struct efx_tx_queue *tx_queue) while (tx_queue->insert_count != tx_queue->write_count) { --tx_queue->insert_count; buffer = &tx_queue->buffer[tx_queue->insert_count & - tx_queue->efx->type->txd_ring_mask]; + EFX_TXQ_MASK]; efx_tsoh_free(tx_queue, buffer); EFX_BUG_ON_PARANOID(buffer->skb); buffer->len = 0; @@ -1099,7 +1085,7 @@ static int efx_enqueue_skb_tso(struct efx_tx_queue *tx_queue, mem_err: EFX_ERR(efx, "Out of memory for TSO headers, or PCI mapping error\n"); - dev_kfree_skb_any((struct sk_buff *)skb); + dev_kfree_skb_any(skb); goto unwind; stop: @@ -1135,7 +1121,7 @@ static void efx_fini_tso(struct efx_tx_queue *tx_queue) unsigned i; if (tx_queue->buffer) { - for (i = 0; i <= tx_queue->efx->type->txd_ring_mask; ++i) + for (i = 0; i <= EFX_TXQ_MASK; ++i) efx_tsoh_free(tx_queue, &tx_queue->buffer[i]); } |