From f610b74b14d74a069f61583131e689550fd5bab3 Mon Sep 17 00:00:00 2001 From: "David S. Miller" Date: Mon, 11 Jul 2011 01:37:28 -0700 Subject: ipv4: Use universal hash for ARP. We need to make sure the multiplier is odd. Signed-off-by: David S. Miller --- include/net/arp.h | 7 +++++++ 1 file changed, 7 insertions(+) (limited to 'include/net/arp.h') diff --git a/include/net/arp.h b/include/net/arp.h index 91f0568a04e..723bde501c6 100644 --- a/include/net/arp.h +++ b/include/net/arp.h @@ -8,6 +8,13 @@ extern struct neigh_table arp_tbl; +static inline u32 arp_hashfn(u32 key, const struct net_device *dev, u32 hash_rnd) +{ + u32 val = key ^ dev->ifindex; + + return val * hash_rnd; +} + extern void arp_init(void); extern int arp_find(unsigned char *haddr, struct sk_buff *skb); extern int arp_ioctl(struct net *net, unsigned int cmd, void __user *arg); -- cgit v1.2.3-70-g09d2 From 3769cffb1c48f64640ffab7ce3bffe867342c0f0 Mon Sep 17 00:00:00 2001 From: David Miller Date: Mon, 11 Jul 2011 22:44:24 +0000 Subject: ipv4: Inline neigh binding. Get rid of all of the useless and costly indirection by doing the neigh hash table lookup directly inside of the neighbour binding. Rename from arp_bind_neighbour to rt_bind_neighbour. Use new helpers {__,}ipv4_neigh_lookup() In rt_bind_neighbour() get rid of useless tests which are never true in the context this function is called, namely dev is never NULL and the dst->neighbour is always NULL. Signed-off-by: David S. Miller --- include/net/arp.h | 33 ++++++++++++++++++++++++++++++++- net/ipv4/arp.c | 24 ------------------------ net/ipv4/route.c | 30 +++++++++++++++++++++++++++--- 3 files changed, 59 insertions(+), 28 deletions(-) (limited to 'include/net/arp.h') diff --git a/include/net/arp.h b/include/net/arp.h index 723bde501c6..5e669e6ffb4 100644 --- a/include/net/arp.h +++ b/include/net/arp.h @@ -15,6 +15,38 @@ static inline u32 arp_hashfn(u32 key, const struct net_device *dev, u32 hash_rnd return val * hash_rnd; } +static inline struct neighbour *__ipv4_neigh_lookup(struct neigh_table *tbl, struct net_device *dev, u32 key) +{ + struct neigh_hash_table *nht; + struct neighbour *n; + u32 hash_val; + + rcu_read_lock_bh(); + nht = rcu_dereference_bh(tbl->nht); + hash_val = arp_hashfn(key, dev, nht->hash_rnd) >> (32 - nht->hash_shift); + for (n = rcu_dereference_bh(nht->hash_buckets[hash_val]); + n != NULL; + n = rcu_dereference_bh(n->next)) { + if (n->dev == dev && *(u32 *)n->primary_key == key) { + if (!atomic_inc_not_zero(&n->refcnt)) + n = NULL; + break; + } + } + rcu_read_unlock_bh(); + + return n; +} + +static inline struct neighbour *ipv4_neigh_lookup(struct neigh_table *tbl, struct net_device *dev, const __be32 *pkey) +{ + struct neighbour *n = __ipv4_neigh_lookup(tbl, dev, + *(__force u32 *)pkey); + if (n) + return n; + return neigh_create(tbl, pkey, dev); +} + extern void arp_init(void); extern int arp_find(unsigned char *haddr, struct sk_buff *skb); extern int arp_ioctl(struct net *net, unsigned int cmd, void __user *arg); @@ -22,7 +54,6 @@ extern void arp_send(int type, int ptype, __be32 dest_ip, struct net_device *dev, __be32 src_ip, const unsigned char *dest_hw, const unsigned char *src_hw, const unsigned char *th); -extern int arp_bind_neighbour(struct dst_entry *dst); extern int arp_mc_map(__be32 addr, u8 *haddr, struct net_device *dev, int dir); extern void arp_ifdown(struct net_device *dev); diff --git a/net/ipv4/arp.c b/net/ipv4/arp.c index 4412b57f6ff..3e5545675cc 100644 --- a/net/ipv4/arp.c +++ b/net/ipv4/arp.c @@ -517,30 +517,6 @@ EXPORT_SYMBOL(arp_find); /* END OF OBSOLETE FUNCTIONS */ -int arp_bind_neighbour(struct dst_entry *dst) -{ - struct net_device *dev = dst->dev; - struct neighbour *n = dst->neighbour; - - if (dev == NULL) - return -EINVAL; - if (n == NULL) { - __be32 nexthop = ((struct rtable *)dst)->rt_gateway; - if (dev->flags & (IFF_LOOPBACK | IFF_POINTOPOINT)) - nexthop = 0; - n = __neigh_lookup_errno( -#if defined(CONFIG_ATM_CLIP) || defined(CONFIG_ATM_CLIP_MODULE) - dev->type == ARPHRD_ATM ? - clip_tbl_hook : -#endif - &arp_tbl, &nexthop, dev); - if (IS_ERR(n)) - return PTR_ERR(n); - dst->neighbour = n; - } - return 0; -} - /* * Check if we can use proxy ARP for this path */ diff --git a/net/ipv4/route.c b/net/ipv4/route.c index a8ccd9bca09..c6388e825ed 100644 --- a/net/ipv4/route.c +++ b/net/ipv4/route.c @@ -108,6 +108,7 @@ #ifdef CONFIG_SYSCTL #include #endif +#include #define RT_FL_TOS(oldflp4) \ ((u32)(oldflp4->flowi4_tos & (IPTOS_RT_MASK | RTO_ONLINK))) @@ -1006,6 +1007,29 @@ static int slow_chain_length(const struct rtable *head) return length >> FRACT_BITS; } +static int rt_bind_neighbour(struct rtable *rt) +{ + static const __be32 inaddr_any = 0; + struct net_device *dev = rt->dst.dev; + struct neigh_table *tbl = &arp_tbl; + const __be32 *nexthop; + struct neighbour *n; + +#if defined(CONFIG_ATM_CLIP) || defined(CONFIG_ATM_CLIP_MODULE) + if (dev->type == ARPHRD_ATM) + tbl = clip_tbl_hook; +#endif + nexthop = &rt->rt_gateway; + if (dev->flags & (IFF_LOOPBACK | IFF_POINTOPOINT)) + nexthop = &inaddr_any; + n = ipv4_neigh_lookup(tbl, dev, nexthop); + if (IS_ERR(n)) + return PTR_ERR(n); + rt->dst.neighbour = n; + + return 0; +} + static struct rtable *rt_intern_hash(unsigned hash, struct rtable *rt, struct sk_buff *skb, int ifindex) { @@ -1042,7 +1066,7 @@ restart: rt->dst.flags |= DST_NOCACHE; if (rt->rt_type == RTN_UNICAST || rt_is_output_route(rt)) { - int err = arp_bind_neighbour(&rt->dst); + int err = rt_bind_neighbour(rt); if (err) { if (net_ratelimit()) printk(KERN_WARNING @@ -1138,7 +1162,7 @@ restart: route or unicast forwarding path. */ if (rt->rt_type == RTN_UNICAST || rt_is_output_route(rt)) { - int err = arp_bind_neighbour(&rt->dst); + int err = rt_bind_neighbour(rt); if (err) { spin_unlock_bh(rt_hash_lock_addr(hash)); @@ -1599,7 +1623,7 @@ static int check_peer_redir(struct dst_entry *dst, struct inet_peer *peer) rt->dst.neighbour = NULL; rt->rt_gateway = peer->redirect_learned.a4; - if (arp_bind_neighbour(&rt->dst) || + if (rt_bind_neighbour(rt) || !(rt->dst.neighbour->nud_state & NUD_VALID)) { if (rt->dst.neighbour) neigh_event_send(rt->dst.neighbour, NULL); -- cgit v1.2.3-70-g09d2 From d3aaeb38c40e5a6c08dd31a1b64da65c4352be36 Mon Sep 17 00:00:00 2001 From: "David S. Miller" Date: Mon, 18 Jul 2011 00:40:17 -0700 Subject: net: Add ->neigh_lookup() operation to dst_ops In the future dst entries will be neigh-less. In that environment we need to have an easy transition point for current users of dst->neighbour outside of the packet output fast path. Signed-off-by: David S. Miller --- include/net/arp.h | 9 --------- include/net/dst.h | 5 +++++ include/net/dst_ops.h | 1 + net/bridge/br_netfilter.c | 6 ++++++ net/decnet/dn_route.c | 7 +++++++ net/ipv4/route.c | 26 +++++++++++++++++++------- net/ipv6/route.c | 7 +++++++ net/xfrm/xfrm_policy.c | 7 +++++++ 8 files changed, 52 insertions(+), 16 deletions(-) (limited to 'include/net/arp.h') diff --git a/include/net/arp.h b/include/net/arp.h index 5e669e6ffb4..4979af8b155 100644 --- a/include/net/arp.h +++ b/include/net/arp.h @@ -38,15 +38,6 @@ static inline struct neighbour *__ipv4_neigh_lookup(struct neigh_table *tbl, str return n; } -static inline struct neighbour *ipv4_neigh_lookup(struct neigh_table *tbl, struct net_device *dev, const __be32 *pkey) -{ - struct neighbour *n = __ipv4_neigh_lookup(tbl, dev, - *(__force u32 *)pkey); - if (n) - return n; - return neigh_create(tbl, pkey, dev); -} - extern void arp_init(void); extern int arp_find(unsigned char *haddr, struct sk_buff *skb); extern int arp_ioctl(struct net *net, unsigned int cmd, void __user *arg); diff --git a/include/net/dst.h b/include/net/dst.h index 8147206eefb..29e255796ce 100644 --- a/include/net/dst.h +++ b/include/net/dst.h @@ -387,6 +387,11 @@ static inline void dst_confirm(struct dst_entry *dst) } } +static inline struct neighbour *dst_neigh_lookup(const struct dst_entry *dst, const void *daddr) +{ + return dst->ops->neigh_lookup(dst, daddr); +} + static inline void dst_link_failure(struct sk_buff *skb) { struct dst_entry *dst = skb_dst(skb); diff --git a/include/net/dst_ops.h b/include/net/dst_ops.h index dc074632894..9adb99845a5 100644 --- a/include/net/dst_ops.h +++ b/include/net/dst_ops.h @@ -26,6 +26,7 @@ struct dst_ops { void (*link_failure)(struct sk_buff *); void (*update_pmtu)(struct dst_entry *dst, u32 mtu); int (*local_out)(struct sk_buff *skb); + struct neighbour * (*neigh_lookup)(const struct dst_entry *dst, const void *daddr); struct kmem_cache *kmem_cachep; diff --git a/net/bridge/br_netfilter.c b/net/bridge/br_netfilter.c index b1a5f9777b7..d6ec3720c77 100644 --- a/net/bridge/br_netfilter.c +++ b/net/bridge/br_netfilter.c @@ -109,11 +109,17 @@ static u32 *fake_cow_metrics(struct dst_entry *dst, unsigned long old) return NULL; } +static struct neighbour *fake_neigh_lookup(const struct dst_entry *dst, const void *daddr) +{ + return NULL; +} + static struct dst_ops fake_dst_ops = { .family = AF_INET, .protocol = cpu_to_be16(ETH_P_IP), .update_pmtu = fake_update_pmtu, .cow_metrics = fake_cow_metrics, + .neigh_lookup = fake_neigh_lookup, }; /* diff --git a/net/decnet/dn_route.c b/net/decnet/dn_route.c index 9bd45fcb3b8..43450c10022 100644 --- a/net/decnet/dn_route.c +++ b/net/decnet/dn_route.c @@ -116,6 +116,7 @@ static void dn_dst_destroy(struct dst_entry *); static struct dst_entry *dn_dst_negative_advice(struct dst_entry *); static void dn_dst_link_failure(struct sk_buff *); static void dn_dst_update_pmtu(struct dst_entry *dst, u32 mtu); +static struct neighbour *dn_dst_neigh_lookup(const struct dst_entry *dst, const void *daddr); static int dn_route_input(struct sk_buff *); static void dn_run_flush(unsigned long dummy); @@ -139,6 +140,7 @@ static struct dst_ops dn_dst_ops = { .negative_advice = dn_dst_negative_advice, .link_failure = dn_dst_link_failure, .update_pmtu = dn_dst_update_pmtu, + .neigh_lookup = dn_dst_neigh_lookup, }; static void dn_dst_destroy(struct dst_entry *dst) @@ -827,6 +829,11 @@ static unsigned int dn_dst_default_mtu(const struct dst_entry *dst) return dst->dev->mtu; } +static struct neighbour *dn_dst_neigh_lookup(const struct dst_entry *dst, const void *daddr) +{ + return __neigh_lookup_errno(&dn_neigh_table, daddr, dst->dev); +} + static int dn_rt_set_next_hop(struct dn_route *rt, struct dn_fib_res *res) { struct dn_fib_info *fi = res->fi; diff --git a/net/ipv4/route.c b/net/ipv4/route.c index 1d4cd3b4fd6..33137307d52 100644 --- a/net/ipv4/route.c +++ b/net/ipv4/route.c @@ -185,6 +185,8 @@ static u32 *ipv4_cow_metrics(struct dst_entry *dst, unsigned long old) return p; } +static struct neighbour *ipv4_neigh_lookup(const struct dst_entry *dst, const void *daddr); + static struct dst_ops ipv4_dst_ops = { .family = AF_INET, .protocol = cpu_to_be16(ETH_P_IP), @@ -199,6 +201,7 @@ static struct dst_ops ipv4_dst_ops = { .link_failure = ipv4_link_failure, .update_pmtu = ip_rt_update_pmtu, .local_out = __ip_local_out, + .neigh_lookup = ipv4_neigh_lookup, }; #define ECN_OR_COST(class) TC_PRIO_##class @@ -1008,22 +1011,30 @@ static int slow_chain_length(const struct rtable *head) return length >> FRACT_BITS; } -static int rt_bind_neighbour(struct rtable *rt) +static struct neighbour *ipv4_neigh_lookup(const struct dst_entry *dst, const void *daddr) { - static const __be32 inaddr_any = 0; - struct net_device *dev = rt->dst.dev; struct neigh_table *tbl = &arp_tbl; - const __be32 *nexthop; + static const __be32 inaddr_any = 0; + struct net_device *dev = dst->dev; + const __be32 *pkey = daddr; struct neighbour *n; #if defined(CONFIG_ATM_CLIP) || defined(CONFIG_ATM_CLIP_MODULE) if (dev->type == ARPHRD_ATM) tbl = clip_tbl_hook; #endif - nexthop = &rt->rt_gateway; if (dev->flags & (IFF_LOOPBACK | IFF_POINTOPOINT)) - nexthop = &inaddr_any; - n = ipv4_neigh_lookup(tbl, dev, nexthop); + pkey = &inaddr_any; + + n = __ipv4_neigh_lookup(tbl, dev, *(__force u32 *)pkey); + if (n) + return n; + return neigh_create(tbl, pkey, dev); +} + +static int rt_bind_neighbour(struct rtable *rt) +{ + struct neighbour *n = ipv4_neigh_lookup(&rt->dst, &rt->rt_gateway); if (IS_ERR(n)) return PTR_ERR(n); dst_set_neighbour(&rt->dst, n); @@ -2734,6 +2745,7 @@ static struct dst_ops ipv4_dst_blackhole_ops = { .default_advmss = ipv4_default_advmss, .update_pmtu = ipv4_rt_blackhole_update_pmtu, .cow_metrics = ipv4_rt_blackhole_cow_metrics, + .neigh_lookup = ipv4_neigh_lookup, }; struct dst_entry *ipv4_blackhole_route(struct net *net, struct dst_entry *dst_orig) diff --git a/net/ipv6/route.c b/net/ipv6/route.c index 2998cb5be90..ddef80f568b 100644 --- a/net/ipv6/route.c +++ b/net/ipv6/route.c @@ -127,6 +127,11 @@ static u32 *ipv6_cow_metrics(struct dst_entry *dst, unsigned long old) return p; } +static struct neighbour *ip6_neigh_lookup(const struct dst_entry *dst, const void *daddr) +{ + return __neigh_lookup_errno(&nd_tbl, daddr, dst->dev); +} + static struct dst_ops ip6_dst_ops_template = { .family = AF_INET6, .protocol = cpu_to_be16(ETH_P_IPV6), @@ -142,6 +147,7 @@ static struct dst_ops ip6_dst_ops_template = { .link_failure = ip6_link_failure, .update_pmtu = ip6_rt_update_pmtu, .local_out = __ip6_local_out, + .neigh_lookup = ip6_neigh_lookup, }; static unsigned int ip6_blackhole_default_mtu(const struct dst_entry *dst) @@ -168,6 +174,7 @@ static struct dst_ops ip6_dst_blackhole_ops = { .default_advmss = ip6_default_advmss, .update_pmtu = ip6_rt_blackhole_update_pmtu, .cow_metrics = ip6_rt_blackhole_cow_metrics, + .neigh_lookup = ip6_neigh_lookup, }; static const u32 ip6_template_metrics[RTAX_MAX] = { diff --git a/net/xfrm/xfrm_policy.c b/net/xfrm/xfrm_policy.c index 7803eb6af41..94fdcc7f103 100644 --- a/net/xfrm/xfrm_policy.c +++ b/net/xfrm/xfrm_policy.c @@ -2385,6 +2385,11 @@ static unsigned int xfrm_default_mtu(const struct dst_entry *dst) return dst_mtu(dst->path); } +static struct neighbour *xfrm_neigh_lookup(const struct dst_entry *dst, const void *daddr) +{ + return dst_neigh_lookup(dst->path, daddr); +} + int xfrm_policy_register_afinfo(struct xfrm_policy_afinfo *afinfo) { struct net *net; @@ -2410,6 +2415,8 @@ int xfrm_policy_register_afinfo(struct xfrm_policy_afinfo *afinfo) dst_ops->negative_advice = xfrm_negative_advice; if (likely(dst_ops->link_failure == NULL)) dst_ops->link_failure = xfrm_link_failure; + if (likely(dst_ops->neigh_lookup == NULL)) + dst_ops->neigh_lookup = xfrm_neigh_lookup; if (likely(afinfo->garbage_collect == NULL)) afinfo->garbage_collect = __xfrm_garbage_collect; xfrm_policy_afinfo[afinfo->family] = afinfo; -- cgit v1.2.3-70-g09d2