From f86dcc5aa8c7908f2c287e7a211228df599e3e71 Mon Sep 17 00:00:00 2001 From: Eric Dumazet Date: Wed, 7 Oct 2009 00:37:59 +0000 Subject: udp: dynamically size hash tables at boot time UDP_HTABLE_SIZE was initialy defined to 128, which is a bit small for several setups. 4000 active UDP sockets -> 32 sockets per chain in average. An incoming frame has to lookup all sockets to find best match, so long chains hurt latency. Instead of a fixed size hash table that cant be perfect for every needs, let UDP stack choose its table size at boot time like tcp/ip route, using alloc_large_system_hash() helper Add an optional boot parameter, uhash_entries=x so that an admin can force a size between 256 and 65536 if needed, like thash_entries and rhash_entries. dmesg logs two new lines : [ 0.647039] UDP hash table entries: 512 (order: 0, 4096 bytes) [ 0.647099] UDP Lite hash table entries: 512 (order: 0, 4096 bytes) Maximal size on 64bit arches would be 65536 slots, ie 1 MBytes for non debugging spinlocks. Signed-off-by: Eric Dumazet Signed-off-by: David S. Miller --- include/net/udp.h | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) (limited to 'include/net/udp.h') diff --git a/include/net/udp.h b/include/net/udp.h index f98abd2ce70..22aa2e7eb1d 100644 --- a/include/net/udp.h +++ b/include/net/udp.h @@ -54,12 +54,19 @@ struct udp_hslot { struct hlist_nulls_head head; spinlock_t lock; } __attribute__((aligned(2 * sizeof(long)))); + struct udp_table { - struct udp_hslot hash[UDP_HTABLE_SIZE]; + struct udp_hslot *hash; + unsigned int mask; + unsigned int log; }; extern struct udp_table udp_table; -extern void udp_table_init(struct udp_table *); - +extern void udp_table_init(struct udp_table *, const char *); +static inline struct udp_hslot *udp_hashslot(struct udp_table *table, + struct net *net, unsigned num) +{ + return &table->hash[udp_hashfn(net, num, table->mask)]; +} /* Note: this must match 'valbool' in sock_setsockopt */ #define UDP_CSUM_NOXMIT 1 -- cgit v1.2.3-70-g09d2 From fdcc8aa953a1123a289791dd192090651036d593 Mon Sep 17 00:00:00 2001 From: Eric Dumazet Date: Sun, 8 Nov 2009 10:17:05 +0000 Subject: udp: add a counter into udp_hslot Adds a counter in udp_hslot to keep an accurate count of sockets present in chain. This will permit to upcoming UDP lookup algo to chose the shortest chain when secondary hash is added. Signed-off-by: Eric Dumazet Signed-off-by: David S. Miller --- include/net/udp.h | 8 ++++++++ net/ipv4/udp.c | 3 +++ 2 files changed, 11 insertions(+) (limited to 'include/net/udp.h') diff --git a/include/net/udp.h b/include/net/udp.h index 22aa2e7eb1d..9167281e47d 100644 --- a/include/net/udp.h +++ b/include/net/udp.h @@ -50,8 +50,16 @@ struct udp_skb_cb { }; #define UDP_SKB_CB(__skb) ((struct udp_skb_cb *)((__skb)->cb)) +/** + * struct udp_hslot - UDP hash slot + * + * @head: head of list of sockets + * @count: number of sockets in 'head' list + * @lock: spinlock protecting changes to head/count + */ struct udp_hslot { struct hlist_nulls_head head; + int count; spinlock_t lock; } __attribute__((aligned(2 * sizeof(long)))); diff --git a/net/ipv4/udp.c b/net/ipv4/udp.c index d5e75e97651..ffc837643a0 100644 --- a/net/ipv4/udp.c +++ b/net/ipv4/udp.c @@ -218,6 +218,7 @@ found: sk->sk_hash = snum; if (sk_unhashed(sk)) { sk_nulls_add_node_rcu(sk, &hslot->head); + hslot->count++; sock_prot_inuse_add(sock_net(sk), sk->sk_prot, 1); } error = 0; @@ -1053,6 +1054,7 @@ void udp_lib_unhash(struct sock *sk) spin_lock_bh(&hslot->lock); if (sk_nulls_del_node_init_rcu(sk)) { + hslot->count--; inet_sk(sk)->inet_num = 0; sock_prot_inuse_add(sock_net(sk), sk->sk_prot, -1); } @@ -1862,6 +1864,7 @@ void __init udp_table_init(struct udp_table *table, const char *name) } for (i = 0; i <= table->mask; i++) { INIT_HLIST_NULLS_HEAD(&table->hash[i].head, i); + table->hash[i].count = 0; spin_lock_init(&table->hash[i].lock); } } -- cgit v1.2.3-70-g09d2 From 512615b6b843ff3ff5ad583f34c39b3f302f5f26 Mon Sep 17 00:00:00 2001 From: Eric Dumazet Date: Sun, 8 Nov 2009 10:17:58 +0000 Subject: udp: secondary hash on (local port, local address) Extends udp_table to contain a secondary hash table. socket anchor for this second hash is free, because UDP doesnt use skc_bind_node : We define an union to hold both skc_bind_node & a new hlist_nulls_node udp_portaddr_node udp_lib_get_port() inserts sockets into second hash chain (additional cost of one atomic op) udp_lib_unhash() deletes socket from second hash chain (additional cost of one atomic op) Note : No spinlock lockdep annotation is needed, because lock for the secondary hash chain is always get after lock for primary hash chain. Signed-off-by: Eric Dumazet Signed-off-by: David S. Miller --- include/linux/udp.h | 1 + include/net/sock.h | 8 ++++++-- include/net/udp.h | 22 ++++++++++++++++++++-- net/ipv4/udp.c | 31 ++++++++++++++++++++++++++----- 4 files changed, 53 insertions(+), 9 deletions(-) (limited to 'include/net/udp.h') diff --git a/include/linux/udp.h b/include/linux/udp.h index 5b4b5274e68..59f0ddf2d28 100644 --- a/include/linux/udp.h +++ b/include/linux/udp.h @@ -57,6 +57,7 @@ struct udp_sock { struct inet_sock inet; #define udp_port_hash inet.sk.__sk_common.skc_u16hashes[0] #define udp_portaddr_hash inet.sk.__sk_common.skc_u16hashes[1] +#define udp_portaddr_node inet.sk.__sk_common.skc_portaddr_node int pending; /* Any pending frames ? */ unsigned int corkflag; /* Cork is required */ __u16 encap_type; /* Is this an Encapsulation socket? */ diff --git a/include/net/sock.h b/include/net/sock.h index 827366b6268..3f1a4804bb3 100644 --- a/include/net/sock.h +++ b/include/net/sock.h @@ -105,7 +105,7 @@ struct net; /** * struct sock_common - minimal network layer representation of sockets * @skc_node: main hash linkage for various protocol lookup tables - * @skc_nulls_node: main hash linkage for UDP/UDP-Lite protocol + * @skc_nulls_node: main hash linkage for TCP/UDP/UDP-Lite protocol * @skc_refcnt: reference count * @skc_tx_queue_mapping: tx queue number for this connection * @skc_hash: hash value used with various protocol lookup tables @@ -115,6 +115,7 @@ struct net; * @skc_reuse: %SO_REUSEADDR setting * @skc_bound_dev_if: bound device index if != 0 * @skc_bind_node: bind hash linkage for various protocol lookup tables + * @skc_portaddr_node: second hash linkage for UDP/UDP-Lite protocol * @skc_prot: protocol handlers inside a network family * @skc_net: reference to the network namespace of this socket * @@ -140,7 +141,10 @@ struct sock_common { volatile unsigned char skc_state; unsigned char skc_reuse; int skc_bound_dev_if; - struct hlist_node skc_bind_node; + union { + struct hlist_node skc_bind_node; + struct hlist_nulls_node skc_portaddr_node; + }; struct proto *skc_prot; #ifdef CONFIG_NET_NS struct net *skc_net; diff --git a/include/net/udp.h b/include/net/udp.h index 9167281e47d..af41850f742 100644 --- a/include/net/udp.h +++ b/include/net/udp.h @@ -63,10 +63,19 @@ struct udp_hslot { spinlock_t lock; } __attribute__((aligned(2 * sizeof(long)))); +/** + * struct udp_table - UDP table + * + * @hash: hash table, sockets are hashed on (local port) + * @hash2: hash table, sockets are hashed on (local port, local address) + * @mask: number of slots in hash tables, minus 1 + * @log: log2(number of slots in hash table) + */ struct udp_table { struct udp_hslot *hash; - unsigned int mask; - unsigned int log; + struct udp_hslot *hash2; + unsigned int mask; + unsigned int log; }; extern struct udp_table udp_table; extern void udp_table_init(struct udp_table *, const char *); @@ -75,6 +84,15 @@ static inline struct udp_hslot *udp_hashslot(struct udp_table *table, { return &table->hash[udp_hashfn(net, num, table->mask)]; } +/* + * For secondary hash, net_hash_mix() is performed before calling + * udp_hashslot2(), this explains difference with udp_hashslot() + */ +static inline struct udp_hslot *udp_hashslot2(struct udp_table *table, + unsigned int hash) +{ + return &table->hash2[hash & table->mask]; +} /* Note: this must match 'valbool' in sock_setsockopt */ #define UDP_CSUM_NOXMIT 1 diff --git a/net/ipv4/udp.c b/net/ipv4/udp.c index af72de1c869..5f04216f35c 100644 --- a/net/ipv4/udp.c +++ b/net/ipv4/udp.c @@ -163,7 +163,7 @@ int udp_lib_get_port(struct sock *sk, unsigned short snum, int (*saddr_comp)(const struct sock *sk1, const struct sock *sk2)) { - struct udp_hslot *hslot; + struct udp_hslot *hslot, *hslot2; struct udp_table *udptable = sk->sk_prot->h.udp_table; int error = 1; struct net *net = sock_net(sk); @@ -222,6 +222,13 @@ found: sk_nulls_add_node_rcu(sk, &hslot->head); hslot->count++; sock_prot_inuse_add(sock_net(sk), sk->sk_prot, 1); + + hslot2 = udp_hashslot2(udptable, udp_sk(sk)->udp_portaddr_hash); + spin_lock(&hslot2->lock); + hlist_nulls_add_head_rcu(&udp_sk(sk)->udp_portaddr_node, + &hslot2->head); + hslot2->count++; + spin_unlock(&hslot2->lock); } error = 0; fail_unlock: @@ -1062,14 +1069,22 @@ void udp_lib_unhash(struct sock *sk) { if (sk_hashed(sk)) { struct udp_table *udptable = sk->sk_prot->h.udp_table; - struct udp_hslot *hslot = udp_hashslot(udptable, sock_net(sk), - udp_sk(sk)->udp_port_hash); + struct udp_hslot *hslot, *hslot2; + + hslot = udp_hashslot(udptable, sock_net(sk), + udp_sk(sk)->udp_port_hash); + hslot2 = udp_hashslot2(udptable, udp_sk(sk)->udp_portaddr_hash); spin_lock_bh(&hslot->lock); if (sk_nulls_del_node_init_rcu(sk)) { hslot->count--; inet_sk(sk)->inet_num = 0; sock_prot_inuse_add(sock_net(sk), sk->sk_prot, -1); + + spin_lock(&hslot2->lock); + hlist_nulls_del_init_rcu(&udp_sk(sk)->udp_portaddr_node); + hslot2->count--; + spin_unlock(&hslot2->lock); } spin_unlock_bh(&hslot->lock); } @@ -1857,7 +1872,7 @@ void __init udp_table_init(struct udp_table *table, const char *name) if (!CONFIG_BASE_SMALL) table->hash = alloc_large_system_hash(name, - sizeof(struct udp_hslot), + 2 * sizeof(struct udp_hslot), uhash_entries, 21, /* one slot per 2 MB */ 0, @@ -1869,17 +1884,23 @@ void __init udp_table_init(struct udp_table *table, const char *name) */ if (CONFIG_BASE_SMALL || table->mask < UDP_HTABLE_SIZE_MIN - 1) { table->hash = kmalloc(UDP_HTABLE_SIZE_MIN * - sizeof(struct udp_hslot), GFP_KERNEL); + 2 * sizeof(struct udp_hslot), GFP_KERNEL); if (!table->hash) panic(name); table->log = ilog2(UDP_HTABLE_SIZE_MIN); table->mask = UDP_HTABLE_SIZE_MIN - 1; } + table->hash2 = table->hash + (table->mask + 1); for (i = 0; i <= table->mask; i++) { INIT_HLIST_NULLS_HEAD(&table->hash[i].head, i); table->hash[i].count = 0; spin_lock_init(&table->hash[i].lock); } + for (i = 0; i <= table->mask; i++) { + INIT_HLIST_NULLS_HEAD(&table->hash2[i].head, i); + table->hash2[i].count = 0; + spin_lock_init(&table->hash2[i].lock); + } } void __init udp_init(void) -- cgit v1.2.3-70-g09d2 From 30fff9231fad757c061285e347b33c5149c2c2e4 Mon Sep 17 00:00:00 2001 From: Eric Dumazet Date: Mon, 9 Nov 2009 05:26:33 +0000 Subject: udp: bind() optimisation UDP bind() can be O(N^2) in some pathological cases. Thanks to secondary hash tables, we can make it O(N) Signed-off-by: Eric Dumazet Signed-off-by: David S. Miller --- include/linux/udp.h | 6 +++++ include/net/udp.h | 3 ++- net/ipv4/udp.c | 73 +++++++++++++++++++++++++++++++++++++++++++++++------ net/ipv6/udp.c | 14 +++++----- 4 files changed, 80 insertions(+), 16 deletions(-) (limited to 'include/net/udp.h') diff --git a/include/linux/udp.h b/include/linux/udp.h index 59f0ddf2d28..03f72a2ba02 100644 --- a/include/linux/udp.h +++ b/include/linux/udp.h @@ -88,6 +88,12 @@ static inline struct udp_sock *udp_sk(const struct sock *sk) return (struct udp_sock *)sk; } +#define udp_portaddr_for_each_entry(__sk, node, list) \ + hlist_nulls_for_each_entry(__sk, node, list, __sk_common.skc_portaddr_node) + +#define udp_portaddr_for_each_entry_rcu(__sk, node, list) \ + hlist_nulls_for_each_entry_rcu(__sk, node, list, __sk_common.skc_portaddr_node) + #define IS_UDPLITE(__sk) (udp_sk(__sk)->pcflag) #endif diff --git a/include/net/udp.h b/include/net/udp.h index af41850f742..5348d80b25b 100644 --- a/include/net/udp.h +++ b/include/net/udp.h @@ -158,7 +158,8 @@ static inline void udp_lib_close(struct sock *sk, long timeout) } extern int udp_lib_get_port(struct sock *sk, unsigned short snum, - int (*)(const struct sock*,const struct sock*)); + int (*)(const struct sock *,const struct sock *), + unsigned int hash2_nulladdr); /* net/ipv4/udp.c */ extern int udp_get_port(struct sock *sk, unsigned short snum, diff --git a/net/ipv4/udp.c b/net/ipv4/udp.c index d73e9170536..1eaf57567eb 100644 --- a/net/ipv4/udp.c +++ b/net/ipv4/udp.c @@ -152,16 +152,49 @@ static int udp_lib_lport_inuse(struct net *net, __u16 num, return 0; } +/* + * Note: we still hold spinlock of primary hash chain, so no other writer + * can insert/delete a socket with local_port == num + */ +static int udp_lib_lport_inuse2(struct net *net, __u16 num, + struct udp_hslot *hslot2, + struct sock *sk, + int (*saddr_comp)(const struct sock *sk1, + const struct sock *sk2)) +{ + struct sock *sk2; + struct hlist_nulls_node *node; + int res = 0; + + spin_lock(&hslot2->lock); + udp_portaddr_for_each_entry(sk2, node, &hslot2->head) + if (net_eq(sock_net(sk2), net) && + sk2 != sk && + (udp_sk(sk2)->udp_port_hash == num) && + (!sk2->sk_reuse || !sk->sk_reuse) && + (!sk2->sk_bound_dev_if || !sk->sk_bound_dev_if + || sk2->sk_bound_dev_if == sk->sk_bound_dev_if) && + (*saddr_comp)(sk, sk2)) { + res = 1; + break; + } + spin_unlock(&hslot2->lock); + return res; +} + /** * udp_lib_get_port - UDP/-Lite port lookup for IPv4 and IPv6 * * @sk: socket struct in question * @snum: port number to look up * @saddr_comp: AF-dependent comparison of bound local IP addresses + * @hash2_nulladdr: AF-dependant hash value in secondary hash chains, + * with NULL address */ int udp_lib_get_port(struct sock *sk, unsigned short snum, int (*saddr_comp)(const struct sock *sk1, - const struct sock *sk2)) + const struct sock *sk2), + unsigned int hash2_nulladdr) { struct udp_hslot *hslot, *hslot2; struct udp_table *udptable = sk->sk_prot->h.udp_table; @@ -210,6 +243,30 @@ int udp_lib_get_port(struct sock *sk, unsigned short snum, } else { hslot = udp_hashslot(udptable, net, snum); spin_lock_bh(&hslot->lock); + if (hslot->count > 10) { + int exist; + unsigned int slot2 = udp_sk(sk)->udp_portaddr_hash ^ snum; + + slot2 &= udptable->mask; + hash2_nulladdr &= udptable->mask; + + hslot2 = udp_hashslot2(udptable, slot2); + if (hslot->count < hslot2->count) + goto scan_primary_hash; + + exist = udp_lib_lport_inuse2(net, snum, hslot2, + sk, saddr_comp); + if (!exist && (hash2_nulladdr != slot2)) { + hslot2 = udp_hashslot2(udptable, hash2_nulladdr); + exist = udp_lib_lport_inuse2(net, snum, hslot2, + sk, saddr_comp); + } + if (exist) + goto fail_unlock; + else + goto found; + } +scan_primary_hash: if (udp_lib_lport_inuse(net, snum, hslot, NULL, sk, saddr_comp, 0)) goto fail_unlock; @@ -255,12 +312,14 @@ static unsigned int udp4_portaddr_hash(struct net *net, __be32 saddr, int udp_v4_get_port(struct sock *sk, unsigned short snum) { + unsigned int hash2_nulladdr = + udp4_portaddr_hash(sock_net(sk), INADDR_ANY, snum); + unsigned int hash2_partial = + udp4_portaddr_hash(sock_net(sk), inet_sk(sk)->inet_rcv_saddr, 0); + /* precompute partial secondary hash */ - udp_sk(sk)->udp_portaddr_hash = - udp4_portaddr_hash(sock_net(sk), - inet_sk(sk)->inet_rcv_saddr, - 0); - return udp_lib_get_port(sk, snum, ipv4_rcv_saddr_equal); + udp_sk(sk)->udp_portaddr_hash = hash2_partial; + return udp_lib_get_port(sk, snum, ipv4_rcv_saddr_equal, hash2_nulladdr); } static inline int compute_score(struct sock *sk, struct net *net, __be32 saddr, @@ -336,8 +395,6 @@ static inline int compute_score2(struct sock *sk, struct net *net, return score; } -#define udp_portaddr_for_each_entry_rcu(__sk, node, list) \ - hlist_nulls_for_each_entry_rcu(__sk, node, list, __sk_common.skc_portaddr_node) /* called with read_rcu_lock() */ static struct sock *udp4_lib_lookup2(struct net *net, diff --git a/net/ipv6/udp.c b/net/ipv6/udp.c index 2915e1dad72..f4c85b20005 100644 --- a/net/ipv6/udp.c +++ b/net/ipv6/udp.c @@ -100,12 +100,14 @@ static unsigned int udp6_portaddr_hash(struct net *net, int udp_v6_get_port(struct sock *sk, unsigned short snum) { + unsigned int hash2_nulladdr = + udp6_portaddr_hash(sock_net(sk), &in6addr_any, snum); + unsigned int hash2_partial = + udp6_portaddr_hash(sock_net(sk), &inet6_sk(sk)->rcv_saddr, 0); + /* precompute partial secondary hash */ - udp_sk(sk)->udp_portaddr_hash = - udp6_portaddr_hash(sock_net(sk), - &inet6_sk(sk)->rcv_saddr, - 0); - return udp_lib_get_port(sk, snum, ipv6_rcv_saddr_equal); + udp_sk(sk)->udp_portaddr_hash = hash2_partial; + return udp_lib_get_port(sk, snum, ipv6_rcv_saddr_equal, hash2_nulladdr); } static inline int compute_score(struct sock *sk, struct net *net, @@ -181,8 +183,6 @@ static inline int compute_score2(struct sock *sk, struct net *net, return score; } -#define udp_portaddr_for_each_entry_rcu(__sk, node, list) \ - hlist_nulls_for_each_entry_rcu(__sk, node, list, __sk_common.skc_portaddr_node) /* called with read_rcu_lock() */ static struct sock *udp6_lib_lookup2(struct net *net, -- cgit v1.2.3-70-g09d2