diff options
| -rw-r--r-- | drivers/net/sfc/net_driver.h | 16 | ||||
| -rw-r--r-- | drivers/net/sfc/nic.c | 42 | ||||
| -rw-r--r-- | drivers/net/sfc/tx.c | 12 |
3 files changed, 68 insertions, 2 deletions
diff --git a/drivers/net/sfc/net_driver.h b/drivers/net/sfc/net_driver.h index 60d63711d43..270e217a53f 100644 --- a/drivers/net/sfc/net_driver.h +++ b/drivers/net/sfc/net_driver.h @@ -142,6 +142,12 @@ struct efx_tx_buffer { * @flushed: Used when handling queue flushing * @read_count: Current read pointer. * This is the number of buffers that have been removed from both rings. + * @old_write_count: The value of @write_count when last checked. + * This is here for performance reasons. The xmit path will + * only get the up-to-date value of @write_count if this + * variable indicates that the queue is empty. This is to + * avoid cache-line ping-pong between the xmit path and the + * completion path. * @stopped: Stopped count. * Set if this TX queue is currently stopping its port. * @insert_count: Current insert pointer @@ -163,6 +169,10 @@ struct efx_tx_buffer { * @tso_long_headers: Number of packets with headers too long for standard * blocks * @tso_packets: Number of packets via the TSO xmit path + * @pushes: Number of times the TX push feature has been used + * @empty_read_count: If the completion path has seen the queue as empty + * and the transmission path has not yet checked this, the value of + * @read_count bitwise-added to %EFX_EMPTY_COUNT_VALID; otherwise 0. */ struct efx_tx_queue { /* Members which don't change on the fast path */ @@ -177,6 +187,7 @@ struct efx_tx_queue { /* Members used mainly on the completion path */ unsigned int read_count ____cacheline_aligned_in_smp; + unsigned int old_write_count; int stopped; /* Members used only on the xmit path */ @@ -187,6 +198,11 @@ struct efx_tx_queue { unsigned int tso_bursts; unsigned int tso_long_headers; unsigned int tso_packets; + unsigned int pushes; + + /* Members shared between paths and sometimes updated */ + unsigned int empty_read_count ____cacheline_aligned_in_smp; +#define EFX_EMPTY_COUNT_VALID 0x80000000 }; /** diff --git a/drivers/net/sfc/nic.c b/drivers/net/sfc/nic.c index 9743cff1513..bda6b1bd072 100644 --- a/drivers/net/sfc/nic.c +++ b/drivers/net/sfc/nic.c @@ -362,6 +362,35 @@ static inline void efx_notify_tx_desc(struct efx_tx_queue *tx_queue) FR_AZ_TX_DESC_UPD_DWORD_P0, tx_queue->queue); } +/* Write pointer and first descriptor for TX descriptor ring */ +static inline void efx_push_tx_desc(struct efx_tx_queue *tx_queue, + const efx_qword_t *txd) +{ + unsigned write_ptr; + efx_oword_t reg; + + BUILD_BUG_ON(FRF_AZ_TX_DESC_LBN != 0); + BUILD_BUG_ON(FR_AA_TX_DESC_UPD_KER != FR_BZ_TX_DESC_UPD_P0); + + write_ptr = tx_queue->write_count & tx_queue->ptr_mask; + EFX_POPULATE_OWORD_2(reg, FRF_AZ_TX_DESC_PUSH_CMD, true, + FRF_AZ_TX_DESC_WPTR, write_ptr); + reg.qword[0] = *txd; + efx_writeo_page(tx_queue->efx, ®, + FR_BZ_TX_DESC_UPD_P0, tx_queue->queue); +} + +static inline bool +efx_may_push_tx_desc(struct efx_tx_queue *tx_queue, unsigned int write_count) +{ + unsigned empty_read_count = ACCESS_ONCE(tx_queue->empty_read_count); + + if (empty_read_count == 0) + return false; + + tx_queue->empty_read_count = 0; + return ((empty_read_count ^ write_count) & ~EFX_EMPTY_COUNT_VALID) == 0; +} /* For each entry inserted into the software descriptor ring, create a * descriptor in the hardware TX descriptor ring (in host memory), and @@ -373,6 +402,7 @@ void efx_nic_push_buffers(struct efx_tx_queue *tx_queue) struct efx_tx_buffer *buffer; efx_qword_t *txd; unsigned write_ptr; + unsigned old_write_count = tx_queue->write_count; BUG_ON(tx_queue->write_count == tx_queue->insert_count); @@ -391,7 +421,15 @@ void efx_nic_push_buffers(struct efx_tx_queue *tx_queue) } while (tx_queue->write_count != tx_queue->insert_count); wmb(); /* Ensure descriptors are written before they are fetched */ - efx_notify_tx_desc(tx_queue); + + if (efx_may_push_tx_desc(tx_queue, old_write_count)) { + txd = efx_tx_desc(tx_queue, + old_write_count & tx_queue->ptr_mask); + efx_push_tx_desc(tx_queue, txd); + ++tx_queue->pushes; + } else { + efx_notify_tx_desc(tx_queue); + } } /* Allocate hardware resources for a TX queue */ @@ -1626,7 +1664,7 @@ void efx_nic_init_common(struct efx_nic *efx) EFX_SET_OWORD_FIELD(temp, FRF_AZ_TX_RX_SPACER, 0xfe); EFX_SET_OWORD_FIELD(temp, FRF_AZ_TX_RX_SPACER_EN, 1); EFX_SET_OWORD_FIELD(temp, FRF_AZ_TX_ONE_PKT_PER_Q, 1); - EFX_SET_OWORD_FIELD(temp, FRF_AZ_TX_PUSH_EN, 0); + EFX_SET_OWORD_FIELD(temp, FRF_AZ_TX_PUSH_EN, 1); EFX_SET_OWORD_FIELD(temp, FRF_AZ_TX_DIS_NON_IP_EV, 1); /* Enable SW_EV to inherit in char driver - assume harmless here */ EFX_SET_OWORD_FIELD(temp, FRF_AZ_TX_SOFT_EVT_EN, 1); diff --git a/drivers/net/sfc/tx.c b/drivers/net/sfc/tx.c index fef22351ddb..bdb92b4af68 100644 --- a/drivers/net/sfc/tx.c +++ b/drivers/net/sfc/tx.c @@ -428,6 +428,16 @@ void efx_xmit_done(struct efx_tx_queue *tx_queue, unsigned int index) __netif_tx_unlock(queue); } } + + /* Check whether the hardware queue is now empty */ + if ((int)(tx_queue->read_count - tx_queue->old_write_count) >= 0) { + tx_queue->old_write_count = ACCESS_ONCE(tx_queue->write_count); + if (tx_queue->read_count == tx_queue->old_write_count) { + smp_mb(); + tx_queue->empty_read_count = + tx_queue->read_count | EFX_EMPTY_COUNT_VALID; + } + } } int efx_probe_tx_queue(struct efx_tx_queue *tx_queue) @@ -473,8 +483,10 @@ void efx_init_tx_queue(struct efx_tx_queue *tx_queue) tx_queue->insert_count = 0; tx_queue->write_count = 0; + tx_queue->old_write_count = 0; tx_queue->read_count = 0; tx_queue->old_read_count = 0; + tx_queue->empty_read_count = 0 | EFX_EMPTY_COUNT_VALID; BUG_ON(tx_queue->stopped); /* Set up TX descriptor ring */ |