From d30e383bb856f614ddb5bbbb5a7d3f86240e41ec Mon Sep 17 00:00:00 2001 From: Eliezer Tamir Date: Mon, 10 Jun 2013 11:40:10 +0300 Subject: tcp: add low latency socket poll support. Adds low latency socket poll support for TCP. In tcp_v[46]_rcv() add a call to sk_mark_ll() to copy the napi_id from the skb to the sk. In tcp_recvmsg(), when there is no data in the socket we busy-poll. This is a good example of how to add busy-poll support to more protocols. Signed-off-by: Alexander Duyck Signed-off-by: Jesse Brandeburg Signed-off-by: Eliezer Tamir Acked-by: Eric Dumazet Tested-by: Willem de Bruijn Signed-off-by: David S. Miller --- net/ipv6/tcp_ipv6.c | 2 ++ 1 file changed, 2 insertions(+) (limited to 'net/ipv6/tcp_ipv6.c') diff --git a/net/ipv6/tcp_ipv6.c b/net/ipv6/tcp_ipv6.c index 0a17ed9eaf3..5cffa5c3e6b 100644 --- a/net/ipv6/tcp_ipv6.c +++ b/net/ipv6/tcp_ipv6.c @@ -63,6 +63,7 @@ #include #include #include +#include #include @@ -1498,6 +1499,7 @@ process: if (sk_filter(sk, skb)) goto discard_and_relse; + sk_mark_ll(sk, skb); skb->dev = NULL; bh_lock_sock_nested(sk); -- cgit v1.2.3-70-g09d2 From 076bb0c82a44fbe46fe2c8527a5b5b64b69f679d Mon Sep 17 00:00:00 2001 From: Eliezer Tamir Date: Wed, 10 Jul 2013 17:13:17 +0300 Subject: net: rename include/net/ll_poll.h to include/net/busy_poll.h Rename the file and correct all the places where it is included. Signed-off-by: Eliezer Tamir Signed-off-by: David S. Miller --- drivers/net/ethernet/broadcom/bnx2x/bnx2x_cmn.c | 2 +- drivers/net/ethernet/intel/ixgbe/ixgbe.h | 2 +- drivers/net/ethernet/mellanox/mlx4/en_netdev.c | 2 +- drivers/net/ethernet/mellanox/mlx4/en_rx.c | 2 +- fs/select.c | 2 +- include/net/busy_poll.h | 183 ++++++++++++++++++++++++ include/net/ll_poll.h | 183 ------------------------ net/core/datagram.c | 2 +- net/core/sock.c | 2 +- net/core/sysctl_net_core.c | 2 +- net/ipv4/tcp.c | 2 +- net/ipv4/tcp_ipv4.c | 2 +- net/ipv4/udp.c | 2 +- net/ipv6/tcp_ipv6.c | 2 +- net/ipv6/udp.c | 2 +- net/socket.c | 2 +- 16 files changed, 197 insertions(+), 197 deletions(-) create mode 100644 include/net/busy_poll.h delete mode 100644 include/net/ll_poll.h (limited to 'net/ipv6/tcp_ipv6.c') diff --git a/drivers/net/ethernet/broadcom/bnx2x/bnx2x_cmn.c b/drivers/net/ethernet/broadcom/bnx2x/bnx2x_cmn.c index ec3aa1d451e..05b6b4e8b07 100644 --- a/drivers/net/ethernet/broadcom/bnx2x/bnx2x_cmn.c +++ b/drivers/net/ethernet/broadcom/bnx2x/bnx2x_cmn.c @@ -24,7 +24,7 @@ #include #include #include -#include +#include #include #include "bnx2x_cmn.h" #include "bnx2x_init.h" diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe.h b/drivers/net/ethernet/intel/ixgbe/ixgbe.h index fb098b46c6a..7be725cdfea 100644 --- a/drivers/net/ethernet/intel/ixgbe/ixgbe.h +++ b/drivers/net/ethernet/intel/ixgbe/ixgbe.h @@ -52,7 +52,7 @@ #include #endif -#include +#include #ifdef CONFIG_NET_LL_RX_POLL #define LL_EXTENDED_STATS diff --git a/drivers/net/ethernet/mellanox/mlx4/en_netdev.c b/drivers/net/ethernet/mellanox/mlx4/en_netdev.c index caf20477056..0fb2438dc2c 100644 --- a/drivers/net/ethernet/mellanox/mlx4/en_netdev.c +++ b/drivers/net/ethernet/mellanox/mlx4/en_netdev.c @@ -38,7 +38,7 @@ #include #include #include -#include +#include #include #include diff --git a/drivers/net/ethernet/mellanox/mlx4/en_rx.c b/drivers/net/ethernet/mellanox/mlx4/en_rx.c index 76997b93fdf..90746d37ac9 100644 --- a/drivers/net/ethernet/mellanox/mlx4/en_rx.c +++ b/drivers/net/ethernet/mellanox/mlx4/en_rx.c @@ -31,7 +31,7 @@ * */ -#include +#include #include #include #include diff --git a/fs/select.c b/fs/select.c index f9f49c40cfd..35d4adc749d 100644 --- a/fs/select.c +++ b/fs/select.c @@ -28,7 +28,7 @@ #include #include #include -#include +#include #include diff --git a/include/net/busy_poll.h b/include/net/busy_poll.h new file mode 100644 index 00000000000..76f03408774 --- /dev/null +++ b/include/net/busy_poll.h @@ -0,0 +1,183 @@ +/* + * Low Latency Sockets + * Copyright(c) 2013 Intel Corporation. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA. + * + * Author: Eliezer Tamir + * + * Contact Information: + * e1000-devel Mailing List + */ + +#ifndef _LINUX_NET_LL_POLL_H +#define _LINUX_NET_LL_POLL_H + +#include +#include + +#ifdef CONFIG_NET_LL_RX_POLL + +struct napi_struct; +extern unsigned int sysctl_net_ll_read __read_mostly; +extern unsigned int sysctl_net_ll_poll __read_mostly; + +/* return values from ndo_ll_poll */ +#define LL_FLUSH_FAILED -1 +#define LL_FLUSH_BUSY -2 + +static inline bool net_busy_loop_on(void) +{ + return sysctl_net_ll_poll; +} + +/* a wrapper to make debug_smp_processor_id() happy + * we can use sched_clock() because we don't care much about precision + * we only care that the average is bounded + */ +#ifdef CONFIG_DEBUG_PREEMPT +static inline u64 busy_loop_us_clock(void) +{ + u64 rc; + + preempt_disable_notrace(); + rc = sched_clock(); + preempt_enable_no_resched_notrace(); + + return rc >> 10; +} +#else /* CONFIG_DEBUG_PREEMPT */ +static inline u64 busy_loop_us_clock(void) +{ + return sched_clock() >> 10; +} +#endif /* CONFIG_DEBUG_PREEMPT */ + +static inline unsigned long sk_busy_loop_end_time(struct sock *sk) +{ + return busy_loop_us_clock() + ACCESS_ONCE(sk->sk_ll_usec); +} + +/* in poll/select we use the global sysctl_net_ll_poll value */ +static inline unsigned long busy_loop_end_time(void) +{ + return busy_loop_us_clock() + ACCESS_ONCE(sysctl_net_ll_poll); +} + +static inline bool sk_can_busy_loop(struct sock *sk) +{ + return sk->sk_ll_usec && sk->sk_napi_id && + !need_resched() && !signal_pending(current); +} + + +static inline bool busy_loop_timeout(unsigned long end_time) +{ + unsigned long now = busy_loop_us_clock(); + + return time_after(now, end_time); +} + +/* when used in sock_poll() nonblock is known at compile time to be true + * so the loop and end_time will be optimized out + */ +static inline bool sk_busy_loop(struct sock *sk, int nonblock) +{ + unsigned long end_time = !nonblock ? sk_busy_loop_end_time(sk) : 0; + const struct net_device_ops *ops; + struct napi_struct *napi; + int rc = false; + + /* + * rcu read lock for napi hash + * bh so we don't race with net_rx_action + */ + rcu_read_lock_bh(); + + napi = napi_by_id(sk->sk_napi_id); + if (!napi) + goto out; + + ops = napi->dev->netdev_ops; + if (!ops->ndo_ll_poll) + goto out; + + do { + rc = ops->ndo_ll_poll(napi); + + if (rc == LL_FLUSH_FAILED) + break; /* permanent failure */ + + if (rc > 0) + /* local bh are disabled so it is ok to use _BH */ + NET_ADD_STATS_BH(sock_net(sk), + LINUX_MIB_LOWLATENCYRXPACKETS, rc); + + } while (!nonblock && skb_queue_empty(&sk->sk_receive_queue) && + !need_resched() && !busy_loop_timeout(end_time)); + + rc = !skb_queue_empty(&sk->sk_receive_queue); +out: + rcu_read_unlock_bh(); + return rc; +} + +/* used in the NIC receive handler to mark the skb */ +static inline void skb_mark_ll(struct sk_buff *skb, struct napi_struct *napi) +{ + skb->napi_id = napi->napi_id; +} + +/* used in the protocol hanlder to propagate the napi_id to the socket */ +static inline void sk_mark_ll(struct sock *sk, struct sk_buff *skb) +{ + sk->sk_napi_id = skb->napi_id; +} + +#else /* CONFIG_NET_LL_RX_POLL */ +static inline unsigned long net_busy_loop_on(void) +{ + return 0; +} + +static inline unsigned long busy_loop_end_time(void) +{ + return 0; +} + +static inline bool sk_can_busy_loop(struct sock *sk) +{ + return false; +} + +static inline bool sk_busy_poll(struct sock *sk, int nonblock) +{ + return false; +} + +static inline void skb_mark_ll(struct sk_buff *skb, struct napi_struct *napi) +{ +} + +static inline void sk_mark_ll(struct sock *sk, struct sk_buff *skb) +{ +} + +static inline bool busy_loop_timeout(unsigned long end_time) +{ + return true; +} + +#endif /* CONFIG_NET_LL_RX_POLL */ +#endif /* _LINUX_NET_LL_POLL_H */ diff --git a/include/net/ll_poll.h b/include/net/ll_poll.h deleted file mode 100644 index 76f03408774..00000000000 --- a/include/net/ll_poll.h +++ /dev/null @@ -1,183 +0,0 @@ -/* - * Low Latency Sockets - * Copyright(c) 2013 Intel Corporation. - * - * This program is free software; you can redistribute it and/or modify it - * under the terms and conditions of the GNU General Public License, - * version 2, as published by the Free Software Foundation. - * - * This program is distributed in the hope it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for - * more details. - * - * You should have received a copy of the GNU General Public License along with - * this program; if not, write to the Free Software Foundation, Inc., - * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA. - * - * Author: Eliezer Tamir - * - * Contact Information: - * e1000-devel Mailing List - */ - -#ifndef _LINUX_NET_LL_POLL_H -#define _LINUX_NET_LL_POLL_H - -#include -#include - -#ifdef CONFIG_NET_LL_RX_POLL - -struct napi_struct; -extern unsigned int sysctl_net_ll_read __read_mostly; -extern unsigned int sysctl_net_ll_poll __read_mostly; - -/* return values from ndo_ll_poll */ -#define LL_FLUSH_FAILED -1 -#define LL_FLUSH_BUSY -2 - -static inline bool net_busy_loop_on(void) -{ - return sysctl_net_ll_poll; -} - -/* a wrapper to make debug_smp_processor_id() happy - * we can use sched_clock() because we don't care much about precision - * we only care that the average is bounded - */ -#ifdef CONFIG_DEBUG_PREEMPT -static inline u64 busy_loop_us_clock(void) -{ - u64 rc; - - preempt_disable_notrace(); - rc = sched_clock(); - preempt_enable_no_resched_notrace(); - - return rc >> 10; -} -#else /* CONFIG_DEBUG_PREEMPT */ -static inline u64 busy_loop_us_clock(void) -{ - return sched_clock() >> 10; -} -#endif /* CONFIG_DEBUG_PREEMPT */ - -static inline unsigned long sk_busy_loop_end_time(struct sock *sk) -{ - return busy_loop_us_clock() + ACCESS_ONCE(sk->sk_ll_usec); -} - -/* in poll/select we use the global sysctl_net_ll_poll value */ -static inline unsigned long busy_loop_end_time(void) -{ - return busy_loop_us_clock() + ACCESS_ONCE(sysctl_net_ll_poll); -} - -static inline bool sk_can_busy_loop(struct sock *sk) -{ - return sk->sk_ll_usec && sk->sk_napi_id && - !need_resched() && !signal_pending(current); -} - - -static inline bool busy_loop_timeout(unsigned long end_time) -{ - unsigned long now = busy_loop_us_clock(); - - return time_after(now, end_time); -} - -/* when used in sock_poll() nonblock is known at compile time to be true - * so the loop and end_time will be optimized out - */ -static inline bool sk_busy_loop(struct sock *sk, int nonblock) -{ - unsigned long end_time = !nonblock ? sk_busy_loop_end_time(sk) : 0; - const struct net_device_ops *ops; - struct napi_struct *napi; - int rc = false; - - /* - * rcu read lock for napi hash - * bh so we don't race with net_rx_action - */ - rcu_read_lock_bh(); - - napi = napi_by_id(sk->sk_napi_id); - if (!napi) - goto out; - - ops = napi->dev->netdev_ops; - if (!ops->ndo_ll_poll) - goto out; - - do { - rc = ops->ndo_ll_poll(napi); - - if (rc == LL_FLUSH_FAILED) - break; /* permanent failure */ - - if (rc > 0) - /* local bh are disabled so it is ok to use _BH */ - NET_ADD_STATS_BH(sock_net(sk), - LINUX_MIB_LOWLATENCYRXPACKETS, rc); - - } while (!nonblock && skb_queue_empty(&sk->sk_receive_queue) && - !need_resched() && !busy_loop_timeout(end_time)); - - rc = !skb_queue_empty(&sk->sk_receive_queue); -out: - rcu_read_unlock_bh(); - return rc; -} - -/* used in the NIC receive handler to mark the skb */ -static inline void skb_mark_ll(struct sk_buff *skb, struct napi_struct *napi) -{ - skb->napi_id = napi->napi_id; -} - -/* used in the protocol hanlder to propagate the napi_id to the socket */ -static inline void sk_mark_ll(struct sock *sk, struct sk_buff *skb) -{ - sk->sk_napi_id = skb->napi_id; -} - -#else /* CONFIG_NET_LL_RX_POLL */ -static inline unsigned long net_busy_loop_on(void) -{ - return 0; -} - -static inline unsigned long busy_loop_end_time(void) -{ - return 0; -} - -static inline bool sk_can_busy_loop(struct sock *sk) -{ - return false; -} - -static inline bool sk_busy_poll(struct sock *sk, int nonblock) -{ - return false; -} - -static inline void skb_mark_ll(struct sk_buff *skb, struct napi_struct *napi) -{ -} - -static inline void sk_mark_ll(struct sock *sk, struct sk_buff *skb) -{ -} - -static inline bool busy_loop_timeout(unsigned long end_time) -{ - return true; -} - -#endif /* CONFIG_NET_LL_RX_POLL */ -#endif /* _LINUX_NET_LL_POLL_H */ diff --git a/net/core/datagram.c b/net/core/datagram.c index 6e9ab31e457..8ab48cd8955 100644 --- a/net/core/datagram.c +++ b/net/core/datagram.c @@ -56,7 +56,7 @@ #include #include #include -#include +#include /* * Is a socket 'connection oriented' ? diff --git a/net/core/sock.c b/net/core/sock.c index ab06b719f5b..9bfe83f4d67 100644 --- a/net/core/sock.c +++ b/net/core/sock.c @@ -139,7 +139,7 @@ #include #endif -#include +#include static DEFINE_MUTEX(proto_list_mutex); static LIST_HEAD(proto_list); diff --git a/net/core/sysctl_net_core.c b/net/core/sysctl_net_core.c index afc677eadd9..1a298cb3dae 100644 --- a/net/core/sysctl_net_core.c +++ b/net/core/sysctl_net_core.c @@ -19,7 +19,7 @@ #include #include #include -#include +#include static int one = 1; diff --git a/net/ipv4/tcp.c b/net/ipv4/tcp.c index 15cbfa94bd8..5423223e93c 100644 --- a/net/ipv4/tcp.c +++ b/net/ipv4/tcp.c @@ -279,7 +279,7 @@ #include #include -#include +#include int sysctl_tcp_fin_timeout __read_mostly = TCP_FIN_TIMEOUT; diff --git a/net/ipv4/tcp_ipv4.c b/net/ipv4/tcp_ipv4.c index 35675e46aff..3a261b41a00 100644 --- a/net/ipv4/tcp_ipv4.c +++ b/net/ipv4/tcp_ipv4.c @@ -75,7 +75,7 @@ #include #include #include -#include +#include #include #include diff --git a/net/ipv4/udp.c b/net/ipv4/udp.c index 6b270e53c20..bcc0ff2c16d 100644 --- a/net/ipv4/udp.c +++ b/net/ipv4/udp.c @@ -109,7 +109,7 @@ #include #include #include -#include +#include #include "udp_impl.h" struct udp_table udp_table __read_mostly; diff --git a/net/ipv6/tcp_ipv6.c b/net/ipv6/tcp_ipv6.c index 5cffa5c3e6b..345bd92d4dd 100644 --- a/net/ipv6/tcp_ipv6.c +++ b/net/ipv6/tcp_ipv6.c @@ -63,7 +63,7 @@ #include #include #include -#include +#include #include diff --git a/net/ipv6/udp.c b/net/ipv6/udp.c index b6f31437a1f..40e72034da0 100644 --- a/net/ipv6/udp.c +++ b/net/ipv6/udp.c @@ -46,7 +46,7 @@ #include #include #include -#include +#include #include #include diff --git a/net/socket.c b/net/socket.c index 45afa648364..6a3e9a3f50a 100644 --- a/net/socket.c +++ b/net/socket.c @@ -104,7 +104,7 @@ #include #include #include -#include +#include #ifdef CONFIG_NET_LL_RX_POLL unsigned int sysctl_net_ll_read __read_mostly; -- cgit v1.2.3-70-g09d2 From 8b80cda536ea9bceec0364e897868a30ee13b992 Mon Sep 17 00:00:00 2001 From: Eliezer Tamir Date: Wed, 10 Jul 2013 17:13:26 +0300 Subject: net: rename ll methods to busy-poll Rename ndo_ll_poll to ndo_busy_poll. Rename sk_mark_ll to sk_mark_napi_id. Rename skb_mark_ll to skb_mark_napi_id. Correct all useres of these functions. Update comments and defines in include/net/busy_poll.h Signed-off-by: Eliezer Tamir Signed-off-by: David S. Miller --- drivers/net/ethernet/broadcom/bnx2x/bnx2x_cmn.c | 2 +- drivers/net/ethernet/broadcom/bnx2x/bnx2x_main.c | 2 +- drivers/net/ethernet/intel/ixgbe/ixgbe_main.c | 4 ++-- drivers/net/ethernet/mellanox/mlx4/en_netdev.c | 2 +- drivers/net/ethernet/mellanox/mlx4/en_rx.c | 2 +- include/linux/netdevice.h | 2 +- include/net/busy_poll.h | 22 ++++++++++++---------- net/ipv4/tcp_ipv4.c | 2 +- net/ipv4/udp.c | 2 +- net/ipv6/tcp_ipv6.c | 2 +- net/ipv6/udp.c | 2 +- 11 files changed, 23 insertions(+), 21 deletions(-) (limited to 'net/ipv6/tcp_ipv6.c') diff --git a/drivers/net/ethernet/broadcom/bnx2x/bnx2x_cmn.c b/drivers/net/ethernet/broadcom/bnx2x/bnx2x_cmn.c index 05b6b4e8b07..3353efe7919 100644 --- a/drivers/net/ethernet/broadcom/bnx2x/bnx2x_cmn.c +++ b/drivers/net/ethernet/broadcom/bnx2x/bnx2x_cmn.c @@ -990,7 +990,7 @@ reuse_rx: __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), le16_to_cpu(cqe_fp->vlan_tag)); - skb_mark_ll(skb, &fp->napi); + skb_mark_napi_id(skb, &fp->napi); if (bnx2x_fp_ll_polling(fp)) netif_receive_skb(skb); diff --git a/drivers/net/ethernet/broadcom/bnx2x/bnx2x_main.c b/drivers/net/ethernet/broadcom/bnx2x/bnx2x_main.c index 15a528bda87..e5da07858a2 100644 --- a/drivers/net/ethernet/broadcom/bnx2x/bnx2x_main.c +++ b/drivers/net/ethernet/broadcom/bnx2x/bnx2x_main.c @@ -12027,7 +12027,7 @@ static const struct net_device_ops bnx2x_netdev_ops = { #endif #ifdef CONFIG_NET_LL_RX_POLL - .ndo_ll_poll = bnx2x_low_latency_recv, + .ndo_busy_poll = bnx2x_low_latency_recv, #endif }; diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c index 047ebaaf014..bad8f14b194 100644 --- a/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c +++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c @@ -1978,7 +1978,7 @@ static int ixgbe_clean_rx_irq(struct ixgbe_q_vector *q_vector, } #endif /* IXGBE_FCOE */ - skb_mark_ll(skb, &q_vector->napi); + skb_mark_napi_id(skb, &q_vector->napi); ixgbe_rx_skb(q_vector, skb); /* update budget accounting */ @@ -7228,7 +7228,7 @@ static const struct net_device_ops ixgbe_netdev_ops = { .ndo_poll_controller = ixgbe_netpoll, #endif #ifdef CONFIG_NET_LL_RX_POLL - .ndo_ll_poll = ixgbe_low_latency_recv, + .ndo_busy_poll = ixgbe_low_latency_recv, #endif #ifdef IXGBE_FCOE .ndo_fcoe_ddp_setup = ixgbe_fcoe_ddp_get, diff --git a/drivers/net/ethernet/mellanox/mlx4/en_netdev.c b/drivers/net/ethernet/mellanox/mlx4/en_netdev.c index 0fb2438dc2c..5eac871399d 100644 --- a/drivers/net/ethernet/mellanox/mlx4/en_netdev.c +++ b/drivers/net/ethernet/mellanox/mlx4/en_netdev.c @@ -2141,7 +2141,7 @@ static const struct net_device_ops mlx4_netdev_ops = { .ndo_rx_flow_steer = mlx4_en_filter_rfs, #endif #ifdef CONFIG_NET_LL_RX_POLL - .ndo_ll_poll = mlx4_en_low_latency_recv, + .ndo_busy_poll = mlx4_en_low_latency_recv, #endif }; diff --git a/drivers/net/ethernet/mellanox/mlx4/en_rx.c b/drivers/net/ethernet/mellanox/mlx4/en_rx.c index 90746d37ac9..dec455c8f62 100644 --- a/drivers/net/ethernet/mellanox/mlx4/en_rx.c +++ b/drivers/net/ethernet/mellanox/mlx4/en_rx.c @@ -767,7 +767,7 @@ int mlx4_en_process_rx_cq(struct net_device *dev, struct mlx4_en_cq *cq, int bud timestamp); } - skb_mark_ll(skb, &cq->napi); + skb_mark_napi_id(skb, &cq->napi); /* Push it up the stack */ netif_receive_skb(skb); diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h index bb82871b849..0741a1e919a 100644 --- a/include/linux/netdevice.h +++ b/include/linux/netdevice.h @@ -974,7 +974,7 @@ struct net_device_ops { void (*ndo_netpoll_cleanup)(struct net_device *dev); #endif #ifdef CONFIG_NET_LL_RX_POLL - int (*ndo_ll_poll)(struct napi_struct *dev); + int (*ndo_busy_poll)(struct napi_struct *dev); #endif int (*ndo_set_vf_mac)(struct net_device *dev, int queue, u8 *mac); diff --git a/include/net/busy_poll.h b/include/net/busy_poll.h index 76f03408774..4ff71908fd4 100644 --- a/include/net/busy_poll.h +++ b/include/net/busy_poll.h @@ -1,5 +1,5 @@ /* - * Low Latency Sockets + * net busy poll support * Copyright(c) 2013 Intel Corporation. * * This program is free software; you can redistribute it and/or modify it @@ -21,8 +21,8 @@ * e1000-devel Mailing List */ -#ifndef _LINUX_NET_LL_POLL_H -#define _LINUX_NET_LL_POLL_H +#ifndef _LINUX_NET_BUSY_POLL_H +#define _LINUX_NET_BUSY_POLL_H #include #include @@ -110,11 +110,11 @@ static inline bool sk_busy_loop(struct sock *sk, int nonblock) goto out; ops = napi->dev->netdev_ops; - if (!ops->ndo_ll_poll) + if (!ops->ndo_busy_poll) goto out; do { - rc = ops->ndo_ll_poll(napi); + rc = ops->ndo_busy_poll(napi); if (rc == LL_FLUSH_FAILED) break; /* permanent failure */ @@ -134,13 +134,14 @@ out: } /* used in the NIC receive handler to mark the skb */ -static inline void skb_mark_ll(struct sk_buff *skb, struct napi_struct *napi) +static inline void skb_mark_napi_id(struct sk_buff *skb, + struct napi_struct *napi) { skb->napi_id = napi->napi_id; } /* used in the protocol hanlder to propagate the napi_id to the socket */ -static inline void sk_mark_ll(struct sock *sk, struct sk_buff *skb) +static inline void sk_mark_napi_id(struct sock *sk, struct sk_buff *skb) { sk->sk_napi_id = skb->napi_id; } @@ -166,11 +167,12 @@ static inline bool sk_busy_poll(struct sock *sk, int nonblock) return false; } -static inline void skb_mark_ll(struct sk_buff *skb, struct napi_struct *napi) +static inline void skb_mark_napi_id(struct sk_buff *skb, + struct napi_struct *napi) { } -static inline void sk_mark_ll(struct sock *sk, struct sk_buff *skb) +static inline void sk_mark_napi_id(struct sock *sk, struct sk_buff *skb) { } @@ -180,4 +182,4 @@ static inline bool busy_loop_timeout(unsigned long end_time) } #endif /* CONFIG_NET_LL_RX_POLL */ -#endif /* _LINUX_NET_LL_POLL_H */ +#endif /* _LINUX_NET_BUSY_POLL_H */ diff --git a/net/ipv4/tcp_ipv4.c b/net/ipv4/tcp_ipv4.c index 3a261b41a00..b299da5ff49 100644 --- a/net/ipv4/tcp_ipv4.c +++ b/net/ipv4/tcp_ipv4.c @@ -1994,7 +1994,7 @@ process: if (sk_filter(sk, skb)) goto discard_and_relse; - sk_mark_ll(sk, skb); + sk_mark_napi_id(sk, skb); skb->dev = NULL; bh_lock_sock_nested(sk); diff --git a/net/ipv4/udp.c b/net/ipv4/udp.c index bcc0ff2c16d..a0d7151ffbd 100644 --- a/net/ipv4/udp.c +++ b/net/ipv4/udp.c @@ -1713,7 +1713,7 @@ int __udp4_lib_rcv(struct sk_buff *skb, struct udp_table *udptable, if (sk != NULL) { int ret; - sk_mark_ll(sk, skb); + sk_mark_napi_id(sk, skb); ret = udp_queue_rcv_skb(sk, skb); sock_put(sk); diff --git a/net/ipv6/tcp_ipv6.c b/net/ipv6/tcp_ipv6.c index 345bd92d4dd..6e1649d5853 100644 --- a/net/ipv6/tcp_ipv6.c +++ b/net/ipv6/tcp_ipv6.c @@ -1499,7 +1499,7 @@ process: if (sk_filter(sk, skb)) goto discard_and_relse; - sk_mark_ll(sk, skb); + sk_mark_napi_id(sk, skb); skb->dev = NULL; bh_lock_sock_nested(sk); diff --git a/net/ipv6/udp.c b/net/ipv6/udp.c index 40e72034da0..f4058150262 100644 --- a/net/ipv6/udp.c +++ b/net/ipv6/udp.c @@ -844,7 +844,7 @@ int __udp6_lib_rcv(struct sk_buff *skb, struct udp_table *udptable, if (sk != NULL) { int ret; - sk_mark_ll(sk, skb); + sk_mark_napi_id(sk, skb); ret = udpv6_queue_rcv_skb(sk, skb); sock_put(sk); -- cgit v1.2.3-70-g09d2