From 0bfbedb14a8a96c529341bec88991a92b41fac72 Mon Sep 17 00:00:00 2001 From: Eric Dumazet Date: Mon, 5 Oct 2009 00:11:22 -0700 Subject: tunnels: Optimize tx path We currently dirty a cache line to update tunnel device stats (tx_packets/tx_bytes). We better use the txq->tx_bytes/tx_packets counters that already are present in cpu cache, in the cache line shared with txq->_xmit_lock This patch extends IPTUNNEL_XMIT() macro to use txq pointer provided by the caller. Also &tunnel->dev->stats can be replaced by &dev->stats Signed-off-by: Eric Dumazet Signed-off-by: David S. Miller --- net/ipv4/ipip.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'net/ipv4/ipip.c') diff --git a/net/ipv4/ipip.c b/net/ipv4/ipip.c index 08ccd344de7..6a5539236ab 100644 --- a/net/ipv4/ipip.c +++ b/net/ipv4/ipip.c @@ -390,7 +390,8 @@ static int ipip_rcv(struct sk_buff *skb) static netdev_tx_t ipip_tunnel_xmit(struct sk_buff *skb, struct net_device *dev) { struct ip_tunnel *tunnel = netdev_priv(dev); - struct net_device_stats *stats = &tunnel->dev->stats; + struct net_device_stats *stats = &dev->stats; + struct netdev_queue *txq = netdev_get_tx_queue(dev, 0); struct iphdr *tiph = &tunnel->parms.iph; u8 tos = tunnel->parms.iph.tos; __be16 df = tiph->frag_off; @@ -478,7 +479,7 @@ static netdev_tx_t ipip_tunnel_xmit(struct sk_buff *skb, struct net_device *dev) struct sk_buff *new_skb = skb_realloc_headroom(skb, max_headroom); if (!new_skb) { ip_rt_put(rt); - stats->tx_dropped++; + txq->tx_dropped++; dev_kfree_skb(skb); return NETDEV_TX_OK; } -- cgit v1.2.3-70-g09d2 From 8f95dd63a2ab6fe7243c4f0bd2c3266e3a5525ab Mon Sep 17 00:00:00 2001 From: Eric Dumazet Date: Fri, 23 Oct 2009 05:42:02 +0000 Subject: ipip: convert hash tables locking to RCU IPIP tunnels use one rwlock to protect their hash tables. This locking scheme can be converted to RCU for free, since netdevice already must wait for a RCU grace period at dismantle time. Signed-off-by: Eric Dumazet Signed-off-by: David S. Miller --- net/ipv4/ipip.c | 49 ++++++++++++++++++++++++++++--------------------- 1 file changed, 28 insertions(+), 21 deletions(-) (limited to 'net/ipv4/ipip.c') diff --git a/net/ipv4/ipip.c b/net/ipv4/ipip.c index 6a5539236ab..3bd69988bcc 100644 --- a/net/ipv4/ipip.c +++ b/net/ipv4/ipip.c @@ -134,7 +134,13 @@ static void ipip_fb_tunnel_init(struct net_device *dev); static void ipip_tunnel_init(struct net_device *dev); static void ipip_tunnel_setup(struct net_device *dev); -static DEFINE_RWLOCK(ipip_lock); +/* + * Locking : hash tables are protected by RCU and a spinlock + */ +static DEFINE_SPINLOCK(ipip_lock); + +#define for_each_ip_tunnel_rcu(start) \ + for (t = rcu_dereference(start); t; t = rcu_dereference(t->next)) static struct ip_tunnel * ipip_tunnel_lookup(struct net *net, __be32 remote, __be32 local) @@ -144,20 +150,21 @@ static struct ip_tunnel * ipip_tunnel_lookup(struct net *net, struct ip_tunnel *t; struct ipip_net *ipn = net_generic(net, ipip_net_id); - for (t = ipn->tunnels_r_l[h0^h1]; t; t = t->next) { + for_each_ip_tunnel_rcu(ipn->tunnels_r_l[h0 ^ h1]) if (local == t->parms.iph.saddr && remote == t->parms.iph.daddr && (t->dev->flags&IFF_UP)) return t; - } - for (t = ipn->tunnels_r[h0]; t; t = t->next) { + + for_each_ip_tunnel_rcu(ipn->tunnels_r[h0]) if (remote == t->parms.iph.daddr && (t->dev->flags&IFF_UP)) return t; - } - for (t = ipn->tunnels_l[h1]; t; t = t->next) { + + for_each_ip_tunnel_rcu(ipn->tunnels_l[h1]) if (local == t->parms.iph.saddr && (t->dev->flags&IFF_UP)) return t; - } - if ((t = ipn->tunnels_wc[0]) != NULL && (t->dev->flags&IFF_UP)) + + t = rcu_dereference(ipn->tunnels_wc[0]); + if (t && (t->dev->flags&IFF_UP)) return t; return NULL; } @@ -193,9 +200,9 @@ static void ipip_tunnel_unlink(struct ipip_net *ipn, struct ip_tunnel *t) for (tp = ipip_bucket(ipn, t); *tp; tp = &(*tp)->next) { if (t == *tp) { - write_lock_bh(&ipip_lock); + spin_lock_bh(&ipip_lock); *tp = t->next; - write_unlock_bh(&ipip_lock); + spin_unlock_bh(&ipip_lock); break; } } @@ -205,10 +212,10 @@ static void ipip_tunnel_link(struct ipip_net *ipn, struct ip_tunnel *t) { struct ip_tunnel **tp = ipip_bucket(ipn, t); + spin_lock_bh(&ipip_lock); t->next = *tp; - write_lock_bh(&ipip_lock); - *tp = t; - write_unlock_bh(&ipip_lock); + rcu_assign_pointer(*tp, t); + spin_unlock_bh(&ipip_lock); } static struct ip_tunnel * ipip_tunnel_locate(struct net *net, @@ -267,9 +274,9 @@ static void ipip_tunnel_uninit(struct net_device *dev) struct ipip_net *ipn = net_generic(net, ipip_net_id); if (dev == ipn->fb_tunnel_dev) { - write_lock_bh(&ipip_lock); + spin_lock_bh(&ipip_lock); ipn->tunnels_wc[0] = NULL; - write_unlock_bh(&ipip_lock); + spin_unlock_bh(&ipip_lock); } else ipip_tunnel_unlink(ipn, netdev_priv(dev)); dev_put(dev); @@ -318,7 +325,7 @@ static int ipip_err(struct sk_buff *skb, u32 info) err = -ENOENT; - read_lock(&ipip_lock); + rcu_read_lock(); t = ipip_tunnel_lookup(dev_net(skb->dev), iph->daddr, iph->saddr); if (t == NULL || t->parms.iph.daddr == 0) goto out; @@ -333,7 +340,7 @@ static int ipip_err(struct sk_buff *skb, u32 info) t->err_count = 1; t->err_time = jiffies; out: - read_unlock(&ipip_lock); + rcu_read_unlock(); return err; } @@ -351,11 +358,11 @@ static int ipip_rcv(struct sk_buff *skb) struct ip_tunnel *tunnel; const struct iphdr *iph = ip_hdr(skb); - read_lock(&ipip_lock); + rcu_read_lock(); if ((tunnel = ipip_tunnel_lookup(dev_net(skb->dev), iph->saddr, iph->daddr)) != NULL) { if (!xfrm4_policy_check(NULL, XFRM_POLICY_IN, skb)) { - read_unlock(&ipip_lock); + rcu_read_unlock(); kfree_skb(skb); return 0; } @@ -374,10 +381,10 @@ static int ipip_rcv(struct sk_buff *skb) nf_reset(skb); ipip_ecn_decapsulate(iph, skb); netif_rx(skb); - read_unlock(&ipip_lock); + rcu_read_unlock(); return 0; } - read_unlock(&ipip_lock); + rcu_read_unlock(); return -1; } -- cgit v1.2.3-70-g09d2 From 0694c4c016df34c718b9f9ef6ba5aca2e178163a Mon Sep 17 00:00:00 2001 From: Eric Dumazet Date: Tue, 27 Oct 2009 07:06:59 +0000 Subject: ipip: Optimize multiple unregistration Speedup module unloading by factorizing synchronize_rcu() calls Signed-off-by: Eric Dumazet Signed-off-by: David S. Miller --- net/ipv4/ipip.c | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) (limited to 'net/ipv4/ipip.c') diff --git a/net/ipv4/ipip.c b/net/ipv4/ipip.c index 3bd69988bcc..a2ca53da437 100644 --- a/net/ipv4/ipip.c +++ b/net/ipv4/ipip.c @@ -754,16 +754,19 @@ static struct xfrm_tunnel ipip_handler = { static const char banner[] __initconst = KERN_INFO "IPv4 over IPv4 tunneling driver\n"; -static void ipip_destroy_tunnels(struct ipip_net *ipn) +static void ipip_destroy_tunnels(struct ipip_net *ipn, struct list_head *head) { int prio; for (prio = 1; prio < 4; prio++) { int h; for (h = 0; h < HASH_SIZE; h++) { - struct ip_tunnel *t; - while ((t = ipn->tunnels[prio][h]) != NULL) - unregister_netdevice(t->dev); + struct ip_tunnel *t = ipn->tunnels[prio][h]; + + while (t != NULL) { + unregister_netdevice_queue(t->dev, head); + t = t->next; + } } } } @@ -816,11 +819,13 @@ err_alloc: static void ipip_exit_net(struct net *net) { struct ipip_net *ipn; + LIST_HEAD(list); ipn = net_generic(net, ipip_net_id); rtnl_lock(); - ipip_destroy_tunnels(ipn); - unregister_netdevice(ipn->fb_tunnel_dev); + ipip_destroy_tunnels(ipn, &list); + unregister_netdevice_queue(ipn->fb_tunnel_dev, &list); + unregister_netdevice_many(&list); rtnl_unlock(); kfree(ipn); } -- cgit v1.2.3-70-g09d2 From f99189b186f3922ede4fa33c02f6edc735b8c981 Mon Sep 17 00:00:00 2001 From: Eric Dumazet Date: Tue, 17 Nov 2009 10:42:49 +0000 Subject: netns: net_identifiers should be read_mostly Signed-off-by: Eric Dumazet Signed-off-by: David S. Miller --- drivers/net/bonding/bond_main.c | 2 +- drivers/net/ppp_generic.c | 2 +- drivers/net/pppoe.c | 2 +- drivers/net/pppol2tp.c | 2 +- net/8021q/vlan.c | 2 +- net/ipv4/ip_gre.c | 2 +- net/ipv4/ipip.c | 2 +- net/ipv6/ip6_tunnel.c | 2 +- net/ipv6/sit.c | 2 +- net/key/af_key.c | 2 +- net/netfilter/nf_conntrack_proto_dccp.c | 2 +- net/netfilter/nf_conntrack_proto_gre.c | 2 +- net/phonet/pn_dev.c | 2 +- 13 files changed, 13 insertions(+), 13 deletions(-) (limited to 'net/ipv4/ipip.c') diff --git a/drivers/net/bonding/bond_main.c b/drivers/net/bonding/bond_main.c index ecea6c29413..726bd755338 100644 --- a/drivers/net/bonding/bond_main.c +++ b/drivers/net/bonding/bond_main.c @@ -158,7 +158,7 @@ MODULE_PARM_DESC(fail_over_mac, "For active-backup, do not set all slaves to the static const char * const version = DRV_DESCRIPTION ": v" DRV_VERSION " (" DRV_RELDATE ")\n"; -int bond_net_id; +int bond_net_id __read_mostly; static __be32 arp_target[BOND_MAX_ARP_TARGETS]; static int arp_ip_count; diff --git a/drivers/net/ppp_generic.c b/drivers/net/ppp_generic.c index 965adb6174c..0a56a778af0 100644 --- a/drivers/net/ppp_generic.c +++ b/drivers/net/ppp_generic.c @@ -184,7 +184,7 @@ static atomic_t ppp_unit_count = ATOMIC_INIT(0); static atomic_t channel_count = ATOMIC_INIT(0); /* per-net private data for this module */ -static int ppp_net_id; +static int ppp_net_id __read_mostly; struct ppp_net { /* units to ppp mapping */ struct idr units_idr; diff --git a/drivers/net/pppoe.c b/drivers/net/pppoe.c index 60c8d233209..a1dcba255b0 100644 --- a/drivers/net/pppoe.c +++ b/drivers/net/pppoe.c @@ -97,7 +97,7 @@ static const struct proto_ops pppoe_ops; static struct ppp_channel_ops pppoe_chan_ops; /* per-net private data for this module */ -static int pppoe_net_id; +static int pppoe_net_id __read_mostly; struct pppoe_net { /* * we could use _single_ hash table for all diff --git a/drivers/net/pppol2tp.c b/drivers/net/pppol2tp.c index 849cc9c62c2..442c382c2c8 100644 --- a/drivers/net/pppol2tp.c +++ b/drivers/net/pppol2tp.c @@ -232,7 +232,7 @@ static struct ppp_channel_ops pppol2tp_chan_ops = { pppol2tp_xmit , NULL }; static const struct proto_ops pppol2tp_ops; /* per-net private data for this module */ -static int pppol2tp_net_id; +static int pppol2tp_net_id __read_mostly; struct pppol2tp_net { struct list_head pppol2tp_tunnel_list; rwlock_t pppol2tp_tunnel_list_lock; diff --git a/net/8021q/vlan.c b/net/8021q/vlan.c index 39f8d012010..d9cb020029b 100644 --- a/net/8021q/vlan.c +++ b/net/8021q/vlan.c @@ -41,7 +41,7 @@ /* Global VLAN variables */ -int vlan_net_id; +int vlan_net_id __read_mostly; /* Our listing of VLAN group(s) */ static struct hlist_head vlan_group_hash[VLAN_GRP_HASH_SIZE]; diff --git a/net/ipv4/ip_gre.c b/net/ipv4/ip_gre.c index a7de9e3a8f1..c5f6af5d0f3 100644 --- a/net/ipv4/ip_gre.c +++ b/net/ipv4/ip_gre.c @@ -125,7 +125,7 @@ static int ipgre_tunnel_bind_dev(struct net_device *dev); #define HASH_SIZE 16 -static int ipgre_net_id; +static int ipgre_net_id __read_mostly; struct ipgre_net { struct ip_tunnel *tunnels[4][HASH_SIZE]; diff --git a/net/ipv4/ipip.c b/net/ipv4/ipip.c index c5b1f71c3cd..7242ffcc44e 100644 --- a/net/ipv4/ipip.c +++ b/net/ipv4/ipip.c @@ -119,7 +119,7 @@ #define HASH_SIZE 16 #define HASH(addr) (((__force u32)addr^((__force u32)addr>>4))&0xF) -static int ipip_net_id; +static int ipip_net_id __read_mostly; struct ipip_net { struct ip_tunnel *tunnels_r_l[HASH_SIZE]; struct ip_tunnel *tunnels_r[HASH_SIZE]; diff --git a/net/ipv6/ip6_tunnel.c b/net/ipv6/ip6_tunnel.c index 1d614113a4b..e5c0f6bb831 100644 --- a/net/ipv6/ip6_tunnel.c +++ b/net/ipv6/ip6_tunnel.c @@ -78,7 +78,7 @@ static void ip6_fb_tnl_dev_init(struct net_device *dev); static void ip6_tnl_dev_init(struct net_device *dev); static void ip6_tnl_dev_setup(struct net_device *dev); -static int ip6_tnl_net_id; +static int ip6_tnl_net_id __read_mostly; struct ip6_tnl_net { /* the IPv6 tunnel fallback device */ struct net_device *fb_tnl_dev; diff --git a/net/ipv6/sit.c b/net/ipv6/sit.c index b6e145a673a..d9deaa7753e 100644 --- a/net/ipv6/sit.c +++ b/net/ipv6/sit.c @@ -66,7 +66,7 @@ static void ipip6_fb_tunnel_init(struct net_device *dev); static void ipip6_tunnel_init(struct net_device *dev); static void ipip6_tunnel_setup(struct net_device *dev); -static int sit_net_id; +static int sit_net_id __read_mostly; struct sit_net { struct ip_tunnel *tunnels_r_l[HASH_SIZE]; struct ip_tunnel *tunnels_r[HASH_SIZE]; diff --git a/net/key/af_key.c b/net/key/af_key.c index 86b2c22d091..478c8b32a5f 100644 --- a/net/key/af_key.c +++ b/net/key/af_key.c @@ -35,7 +35,7 @@ #define _X2KEY(x) ((x) == XFRM_INF ? 0 : (x)) #define _KEY2X(x) ((x) == 0 ? XFRM_INF : (x)) -static int pfkey_net_id; +static int pfkey_net_id __read_mostly; struct netns_pfkey { /* List of all pfkey sockets. */ struct hlist_head table; diff --git a/net/netfilter/nf_conntrack_proto_dccp.c b/net/netfilter/nf_conntrack_proto_dccp.c index 1b816a2ea81..80abdf297b3 100644 --- a/net/netfilter/nf_conntrack_proto_dccp.c +++ b/net/netfilter/nf_conntrack_proto_dccp.c @@ -384,7 +384,7 @@ dccp_state_table[CT_DCCP_ROLE_MAX + 1][DCCP_PKT_SYNCACK + 1][CT_DCCP_MAX + 1] = }; /* this module per-net specifics */ -static int dccp_net_id; +static int dccp_net_id __read_mostly; struct dccp_net { int dccp_loose; unsigned int dccp_timeout[CT_DCCP_MAX + 1]; diff --git a/net/netfilter/nf_conntrack_proto_gre.c b/net/netfilter/nf_conntrack_proto_gre.c index a54a0af0edb..91d0e719d67 100644 --- a/net/netfilter/nf_conntrack_proto_gre.c +++ b/net/netfilter/nf_conntrack_proto_gre.c @@ -43,7 +43,7 @@ #define GRE_TIMEOUT (30 * HZ) #define GRE_STREAM_TIMEOUT (180 * HZ) -static int proto_gre_net_id; +static int proto_gre_net_id __read_mostly; struct netns_proto_gre { rwlock_t keymap_lock; struct list_head keymap_list; diff --git a/net/phonet/pn_dev.c b/net/phonet/pn_dev.c index 3287f8f0b5c..d5ad7947d77 100644 --- a/net/phonet/pn_dev.c +++ b/net/phonet/pn_dev.c @@ -43,7 +43,7 @@ struct phonet_net { struct phonet_routes routes; }; -int phonet_net_id; +int phonet_net_id __read_mostly; struct phonet_device_list *phonet_device_list(struct net *net) { -- cgit v1.2.3-70-g09d2 From 86de8a631e90a96d136ffd877719471a0b8d8b6d Mon Sep 17 00:00:00 2001 From: "Eric W. Biederman" Date: Sun, 29 Nov 2009 15:46:14 +0000 Subject: net: Simplify ipip pernet operations. Take advantage of the new pernet automatic storage management, and stop using compatibility network namespace functions. Signed-off-by: Eric W. Biederman Signed-off-by: David S. Miller --- net/ipv4/ipip.c | 24 ++++++------------------ 1 file changed, 6 insertions(+), 18 deletions(-) (limited to 'net/ipv4/ipip.c') diff --git a/net/ipv4/ipip.c b/net/ipv4/ipip.c index 7242ffcc44e..eda04fed337 100644 --- a/net/ipv4/ipip.c +++ b/net/ipv4/ipip.c @@ -775,17 +775,8 @@ static void ipip_destroy_tunnels(struct ipip_net *ipn, struct list_head *head) static int ipip_init_net(struct net *net) { + struct ipip_net *ipn = net_generic(net, ipip_net_id); int err; - struct ipip_net *ipn; - - err = -ENOMEM; - ipn = kzalloc(sizeof(struct ipip_net), GFP_KERNEL); - if (ipn == NULL) - goto err_alloc; - - err = net_assign_generic(net, ipip_net_id, ipn); - if (err < 0) - goto err_assign; ipn->tunnels[0] = ipn->tunnels_wc; ipn->tunnels[1] = ipn->tunnels_l; @@ -812,29 +803,26 @@ err_reg_dev: free_netdev(ipn->fb_tunnel_dev); err_alloc_dev: /* nothing */ -err_assign: - kfree(ipn); -err_alloc: return err; } static void ipip_exit_net(struct net *net) { - struct ipip_net *ipn; + struct ipip_net *ipn = net_generic(net, ipip_net_id); LIST_HEAD(list); - ipn = net_generic(net, ipip_net_id); rtnl_lock(); ipip_destroy_tunnels(ipn, &list); unregister_netdevice_queue(ipn->fb_tunnel_dev, &list); unregister_netdevice_many(&list); rtnl_unlock(); - kfree(ipn); } static struct pernet_operations ipip_net_ops = { .init = ipip_init_net, .exit = ipip_exit_net, + .id = &ipip_net_id, + .size = sizeof(struct ipip_net), }; static int __init ipip_init(void) @@ -848,7 +836,7 @@ static int __init ipip_init(void) return -EAGAIN; } - err = register_pernet_gen_device(&ipip_net_id, &ipip_net_ops); + err = register_pernet_device(&ipip_net_ops); if (err) xfrm4_tunnel_deregister(&ipip_handler, AF_INET); @@ -860,7 +848,7 @@ static void __exit ipip_fini(void) if (xfrm4_tunnel_deregister(&ipip_handler, AF_INET)) printk(KERN_INFO "ipip close: can't deregister tunnel\n"); - unregister_pernet_gen_device(ipip_net_id, &ipip_net_ops); + unregister_pernet_device(&ipip_net_ops); } module_init(ipip_init); -- cgit v1.2.3-70-g09d2