summaryrefslogtreecommitdiffstats
path: root/net/core
diff options
context:
space:
mode:
Diffstat (limited to 'net/core')
-rw-r--r--net/core/datagram.c2
-rw-r--r--net/core/dev.c163
-rw-r--r--net/core/gen_estimator.c12
-rw-r--r--net/core/gen_stats.c11
-rw-r--r--net/core/net_namespace.c2
-rw-r--r--net/core/netpoll.c7
-rw-r--r--net/core/skbuff.c13
-rw-r--r--net/core/sock.c51
8 files changed, 154 insertions, 107 deletions
diff --git a/net/core/datagram.c b/net/core/datagram.c
index 58abee1f1df..b0fe69211ee 100644
--- a/net/core/datagram.c
+++ b/net/core/datagram.c
@@ -712,7 +712,7 @@ unsigned int datagram_poll(struct file *file, struct socket *sock,
struct sock *sk = sock->sk;
unsigned int mask;
- poll_wait(file, sk->sk_sleep, wait);
+ sock_poll_wait(file, sk->sk_sleep, wait);
mask = 0;
/* exceptional events? */
diff --git a/net/core/dev.c b/net/core/dev.c
index 576a61574a9..6a94475aee8 100644
--- a/net/core/dev.c
+++ b/net/core/dev.c
@@ -2310,8 +2310,6 @@ ncls:
if (!skb)
goto out;
- skb_orphan(skb);
-
type = skb->protocol;
list_for_each_entry_rcu(ptype,
&ptype_base[ntohs(type) & PTYPE_HASH_MASK], list) {
@@ -2825,9 +2823,11 @@ static void net_rx_action(struct softirq_action *h)
* move the instance around on the list at-will.
*/
if (unlikely(work == weight)) {
- if (unlikely(napi_disable_pending(n)))
- __napi_complete(n);
- else
+ if (unlikely(napi_disable_pending(n))) {
+ local_irq_enable();
+ napi_complete(n);
+ local_irq_disable();
+ } else
list_move_tail(&n->poll_list, list);
}
@@ -3461,10 +3461,10 @@ void __dev_set_rx_mode(struct net_device *dev)
/* Unicast addresses changes may only happen under the rtnl,
* therefore calling __dev_set_promiscuity here is safe.
*/
- if (dev->uc_count > 0 && !dev->uc_promisc) {
+ if (dev->uc.count > 0 && !dev->uc_promisc) {
__dev_set_promiscuity(dev, 1);
dev->uc_promisc = 1;
- } else if (dev->uc_count == 0 && dev->uc_promisc) {
+ } else if (dev->uc.count == 0 && dev->uc_promisc) {
__dev_set_promiscuity(dev, -1);
dev->uc_promisc = 0;
}
@@ -3483,9 +3483,8 @@ void dev_set_rx_mode(struct net_device *dev)
/* hw addresses list handling functions */
-static int __hw_addr_add(struct list_head *list, int *delta,
- unsigned char *addr, int addr_len,
- unsigned char addr_type)
+static int __hw_addr_add(struct netdev_hw_addr_list *list, unsigned char *addr,
+ int addr_len, unsigned char addr_type)
{
struct netdev_hw_addr *ha;
int alloc_size;
@@ -3493,7 +3492,7 @@ static int __hw_addr_add(struct list_head *list, int *delta,
if (addr_len > MAX_ADDR_LEN)
return -EINVAL;
- list_for_each_entry(ha, list, list) {
+ list_for_each_entry(ha, &list->list, list) {
if (!memcmp(ha->addr, addr, addr_len) &&
ha->type == addr_type) {
ha->refcount++;
@@ -3512,9 +3511,8 @@ static int __hw_addr_add(struct list_head *list, int *delta,
ha->type = addr_type;
ha->refcount = 1;
ha->synced = false;
- list_add_tail_rcu(&ha->list, list);
- if (delta)
- (*delta)++;
+ list_add_tail_rcu(&ha->list, &list->list);
+ list->count++;
return 0;
}
@@ -3526,120 +3524,121 @@ static void ha_rcu_free(struct rcu_head *head)
kfree(ha);
}
-static int __hw_addr_del(struct list_head *list, int *delta,
- unsigned char *addr, int addr_len,
- unsigned char addr_type)
+static int __hw_addr_del(struct netdev_hw_addr_list *list, unsigned char *addr,
+ int addr_len, unsigned char addr_type)
{
struct netdev_hw_addr *ha;
- list_for_each_entry(ha, list, list) {
+ list_for_each_entry(ha, &list->list, list) {
if (!memcmp(ha->addr, addr, addr_len) &&
(ha->type == addr_type || !addr_type)) {
if (--ha->refcount)
return 0;
list_del_rcu(&ha->list);
call_rcu(&ha->rcu_head, ha_rcu_free);
- if (delta)
- (*delta)--;
+ list->count--;
return 0;
}
}
return -ENOENT;
}
-static int __hw_addr_add_multiple(struct list_head *to_list, int *to_delta,
- struct list_head *from_list, int addr_len,
+static int __hw_addr_add_multiple(struct netdev_hw_addr_list *to_list,
+ struct netdev_hw_addr_list *from_list,
+ int addr_len,
unsigned char addr_type)
{
int err;
struct netdev_hw_addr *ha, *ha2;
unsigned char type;
- list_for_each_entry(ha, from_list, list) {
+ list_for_each_entry(ha, &from_list->list, list) {
type = addr_type ? addr_type : ha->type;
- err = __hw_addr_add(to_list, to_delta, ha->addr,
- addr_len, type);
+ err = __hw_addr_add(to_list, ha->addr, addr_len, type);
if (err)
goto unroll;
}
return 0;
unroll:
- list_for_each_entry(ha2, from_list, list) {
+ list_for_each_entry(ha2, &from_list->list, list) {
if (ha2 == ha)
break;
type = addr_type ? addr_type : ha2->type;
- __hw_addr_del(to_list, to_delta, ha2->addr,
- addr_len, type);
+ __hw_addr_del(to_list, ha2->addr, addr_len, type);
}
return err;
}
-static void __hw_addr_del_multiple(struct list_head *to_list, int *to_delta,
- struct list_head *from_list, int addr_len,
+static void __hw_addr_del_multiple(struct netdev_hw_addr_list *to_list,
+ struct netdev_hw_addr_list *from_list,
+ int addr_len,
unsigned char addr_type)
{
struct netdev_hw_addr *ha;
unsigned char type;
- list_for_each_entry(ha, from_list, list) {
+ list_for_each_entry(ha, &from_list->list, list) {
type = addr_type ? addr_type : ha->type;
- __hw_addr_del(to_list, to_delta, ha->addr,
- addr_len, addr_type);
+ __hw_addr_del(to_list, ha->addr, addr_len, addr_type);
}
}
-static int __hw_addr_sync(struct list_head *to_list, int *to_delta,
- struct list_head *from_list, int *from_delta,
+static int __hw_addr_sync(struct netdev_hw_addr_list *to_list,
+ struct netdev_hw_addr_list *from_list,
int addr_len)
{
int err = 0;
struct netdev_hw_addr *ha, *tmp;
- list_for_each_entry_safe(ha, tmp, from_list, list) {
+ list_for_each_entry_safe(ha, tmp, &from_list->list, list) {
if (!ha->synced) {
- err = __hw_addr_add(to_list, to_delta, ha->addr,
+ err = __hw_addr_add(to_list, ha->addr,
addr_len, ha->type);
if (err)
break;
ha->synced = true;
ha->refcount++;
} else if (ha->refcount == 1) {
- __hw_addr_del(to_list, to_delta, ha->addr,
- addr_len, ha->type);
- __hw_addr_del(from_list, from_delta, ha->addr,
- addr_len, ha->type);
+ __hw_addr_del(to_list, ha->addr, addr_len, ha->type);
+ __hw_addr_del(from_list, ha->addr, addr_len, ha->type);
}
}
return err;
}
-static void __hw_addr_unsync(struct list_head *to_list, int *to_delta,
- struct list_head *from_list, int *from_delta,
+static void __hw_addr_unsync(struct netdev_hw_addr_list *to_list,
+ struct netdev_hw_addr_list *from_list,
int addr_len)
{
struct netdev_hw_addr *ha, *tmp;
- list_for_each_entry_safe(ha, tmp, from_list, list) {
+ list_for_each_entry_safe(ha, tmp, &from_list->list, list) {
if (ha->synced) {
- __hw_addr_del(to_list, to_delta, ha->addr,
+ __hw_addr_del(to_list, ha->addr,
addr_len, ha->type);
ha->synced = false;
- __hw_addr_del(from_list, from_delta, ha->addr,
+ __hw_addr_del(from_list, ha->addr,
addr_len, ha->type);
}
}
}
-
-static void __hw_addr_flush(struct list_head *list)
+static void __hw_addr_flush(struct netdev_hw_addr_list *list)
{
struct netdev_hw_addr *ha, *tmp;
- list_for_each_entry_safe(ha, tmp, list, list) {
+ list_for_each_entry_safe(ha, tmp, &list->list, list) {
list_del_rcu(&ha->list);
call_rcu(&ha->rcu_head, ha_rcu_free);
}
+ list->count = 0;
+}
+
+static void __hw_addr_init(struct netdev_hw_addr_list *list)
+{
+ INIT_LIST_HEAD(&list->list);
+ list->count = 0;
}
/* Device addresses handling functions */
@@ -3648,7 +3647,7 @@ static void dev_addr_flush(struct net_device *dev)
{
/* rtnl_mutex must be held here */
- __hw_addr_flush(&dev->dev_addr_list);
+ __hw_addr_flush(&dev->dev_addrs);
dev->dev_addr = NULL;
}
@@ -3660,16 +3659,16 @@ static int dev_addr_init(struct net_device *dev)
/* rtnl_mutex must be held here */
- INIT_LIST_HEAD(&dev->dev_addr_list);
+ __hw_addr_init(&dev->dev_addrs);
memset(addr, 0, sizeof(addr));
- err = __hw_addr_add(&dev->dev_addr_list, NULL, addr, sizeof(addr),
+ err = __hw_addr_add(&dev->dev_addrs, addr, sizeof(addr),
NETDEV_HW_ADDR_T_LAN);
if (!err) {
/*
* Get the first (previously created) address from the list
* and set dev_addr pointer to this location.
*/
- ha = list_first_entry(&dev->dev_addr_list,
+ ha = list_first_entry(&dev->dev_addrs.list,
struct netdev_hw_addr, list);
dev->dev_addr = ha->addr;
}
@@ -3694,8 +3693,7 @@ int dev_addr_add(struct net_device *dev, unsigned char *addr,
ASSERT_RTNL();
- err = __hw_addr_add(&dev->dev_addr_list, NULL, addr, dev->addr_len,
- addr_type);
+ err = __hw_addr_add(&dev->dev_addrs, addr, dev->addr_len, addr_type);
if (!err)
call_netdevice_notifiers(NETDEV_CHANGEADDR, dev);
return err;
@@ -3725,11 +3723,12 @@ int dev_addr_del(struct net_device *dev, unsigned char *addr,
* We can not remove the first address from the list because
* dev->dev_addr points to that.
*/
- ha = list_first_entry(&dev->dev_addr_list, struct netdev_hw_addr, list);
+ ha = list_first_entry(&dev->dev_addrs.list,
+ struct netdev_hw_addr, list);
if (ha->addr == dev->dev_addr && ha->refcount == 1)
return -ENOENT;
- err = __hw_addr_del(&dev->dev_addr_list, NULL, addr, dev->addr_len,
+ err = __hw_addr_del(&dev->dev_addrs, addr, dev->addr_len,
addr_type);
if (!err)
call_netdevice_notifiers(NETDEV_CHANGEADDR, dev);
@@ -3757,8 +3756,7 @@ int dev_addr_add_multiple(struct net_device *to_dev,
if (from_dev->addr_len != to_dev->addr_len)
return -EINVAL;
- err = __hw_addr_add_multiple(&to_dev->dev_addr_list, NULL,
- &from_dev->dev_addr_list,
+ err = __hw_addr_add_multiple(&to_dev->dev_addrs, &from_dev->dev_addrs,
to_dev->addr_len, addr_type);
if (!err)
call_netdevice_notifiers(NETDEV_CHANGEADDR, to_dev);
@@ -3784,15 +3782,14 @@ int dev_addr_del_multiple(struct net_device *to_dev,
if (from_dev->addr_len != to_dev->addr_len)
return -EINVAL;
- __hw_addr_del_multiple(&to_dev->dev_addr_list, NULL,
- &from_dev->dev_addr_list,
+ __hw_addr_del_multiple(&to_dev->dev_addrs, &from_dev->dev_addrs,
to_dev->addr_len, addr_type);
call_netdevice_notifiers(NETDEV_CHANGEADDR, to_dev);
return 0;
}
EXPORT_SYMBOL(dev_addr_del_multiple);
-/* unicast and multicast addresses handling functions */
+/* multicast addresses handling functions */
int __dev_addr_delete(struct dev_addr_list **list, int *count,
void *addr, int alen, int glbl)
@@ -3868,10 +3865,12 @@ int dev_unicast_delete(struct net_device *dev, void *addr)
ASSERT_RTNL();
- err = __hw_addr_del(&dev->uc_list, &dev->uc_count, addr,
- dev->addr_len, NETDEV_HW_ADDR_T_UNICAST);
+ netif_addr_lock_bh(dev);
+ err = __hw_addr_del(&dev->uc, addr, dev->addr_len,
+ NETDEV_HW_ADDR_T_UNICAST);
if (!err)
__dev_set_rx_mode(dev);
+ netif_addr_unlock_bh(dev);
return err;
}
EXPORT_SYMBOL(dev_unicast_delete);
@@ -3892,10 +3891,12 @@ int dev_unicast_add(struct net_device *dev, void *addr)
ASSERT_RTNL();
- err = __hw_addr_add(&dev->uc_list, &dev->uc_count, addr,
- dev->addr_len, NETDEV_HW_ADDR_T_UNICAST);
+ netif_addr_lock_bh(dev);
+ err = __hw_addr_add(&dev->uc, addr, dev->addr_len,
+ NETDEV_HW_ADDR_T_UNICAST);
if (!err)
__dev_set_rx_mode(dev);
+ netif_addr_unlock_bh(dev);
return err;
}
EXPORT_SYMBOL(dev_unicast_add);
@@ -3952,7 +3953,8 @@ void __dev_addr_unsync(struct dev_addr_list **to, int *to_count,
* @from: source device
*
* Add newly added addresses to the destination device and release
- * addresses that have no users left.
+ * addresses that have no users left. The source device must be
+ * locked by netif_tx_lock_bh.
*
* This function is intended to be called from the dev->set_rx_mode
* function of layered software devices.
@@ -3961,15 +3963,14 @@ int dev_unicast_sync(struct net_device *to, struct net_device *from)
{
int err = 0;
- ASSERT_RTNL();
-
if (to->addr_len != from->addr_len)
return -EINVAL;
- err = __hw_addr_sync(&to->uc_list, &to->uc_count,
- &from->uc_list, &from->uc_count, to->addr_len);
+ netif_addr_lock_bh(to);
+ err = __hw_addr_sync(&to->uc, &from->uc, to->addr_len);
if (!err)
__dev_set_rx_mode(to);
+ netif_addr_unlock_bh(to);
return err;
}
EXPORT_SYMBOL(dev_unicast_sync);
@@ -3985,30 +3986,28 @@ EXPORT_SYMBOL(dev_unicast_sync);
*/
void dev_unicast_unsync(struct net_device *to, struct net_device *from)
{
- ASSERT_RTNL();
-
if (to->addr_len != from->addr_len)
return;
- __hw_addr_unsync(&to->uc_list, &to->uc_count,
- &from->uc_list, &from->uc_count, to->addr_len);
+ netif_addr_lock_bh(from);
+ netif_addr_lock(to);
+ __hw_addr_unsync(&to->uc, &from->uc, to->addr_len);
__dev_set_rx_mode(to);
+ netif_addr_unlock(to);
+ netif_addr_unlock_bh(from);
}
EXPORT_SYMBOL(dev_unicast_unsync);
static void dev_unicast_flush(struct net_device *dev)
{
- /* rtnl_mutex must be held here */
-
- __hw_addr_flush(&dev->uc_list);
- dev->uc_count = 0;
+ netif_addr_lock_bh(dev);
+ __hw_addr_flush(&dev->uc);
+ netif_addr_unlock_bh(dev);
}
static void dev_unicast_init(struct net_device *dev)
{
- /* rtnl_mutex must be held here */
-
- INIT_LIST_HEAD(&dev->uc_list);
+ __hw_addr_init(&dev->uc);
}
diff --git a/net/core/gen_estimator.c b/net/core/gen_estimator.c
index 78e5bfc454a..493775f4f2f 100644
--- a/net/core/gen_estimator.c
+++ b/net/core/gen_estimator.c
@@ -81,7 +81,7 @@
struct gen_estimator
{
struct list_head list;
- struct gnet_stats_basic *bstats;
+ struct gnet_stats_basic_packed *bstats;
struct gnet_stats_rate_est *rate_est;
spinlock_t *stats_lock;
int ewma_log;
@@ -165,7 +165,7 @@ static void gen_add_node(struct gen_estimator *est)
}
static
-struct gen_estimator *gen_find_node(const struct gnet_stats_basic *bstats,
+struct gen_estimator *gen_find_node(const struct gnet_stats_basic_packed *bstats,
const struct gnet_stats_rate_est *rate_est)
{
struct rb_node *p = est_root.rb_node;
@@ -202,7 +202,7 @@ struct gen_estimator *gen_find_node(const struct gnet_stats_basic *bstats,
*
* NOTE: Called under rtnl_mutex
*/
-int gen_new_estimator(struct gnet_stats_basic *bstats,
+int gen_new_estimator(struct gnet_stats_basic_packed *bstats,
struct gnet_stats_rate_est *rate_est,
spinlock_t *stats_lock,
struct nlattr *opt)
@@ -262,7 +262,7 @@ static void __gen_kill_estimator(struct rcu_head *head)
*
* NOTE: Called under rtnl_mutex
*/
-void gen_kill_estimator(struct gnet_stats_basic *bstats,
+void gen_kill_estimator(struct gnet_stats_basic_packed *bstats,
struct gnet_stats_rate_est *rate_est)
{
struct gen_estimator *e;
@@ -292,7 +292,7 @@ EXPORT_SYMBOL(gen_kill_estimator);
*
* Returns 0 on success or a negative error code.
*/
-int gen_replace_estimator(struct gnet_stats_basic *bstats,
+int gen_replace_estimator(struct gnet_stats_basic_packed *bstats,
struct gnet_stats_rate_est *rate_est,
spinlock_t *stats_lock, struct nlattr *opt)
{
@@ -308,7 +308,7 @@ EXPORT_SYMBOL(gen_replace_estimator);
*
* Returns true if estimator is active, and false if not.
*/
-bool gen_estimator_active(const struct gnet_stats_basic *bstats,
+bool gen_estimator_active(const struct gnet_stats_basic_packed *bstats,
const struct gnet_stats_rate_est *rate_est)
{
ASSERT_RTNL();
diff --git a/net/core/gen_stats.c b/net/core/gen_stats.c
index c3d0ffeac24..8569310268a 100644
--- a/net/core/gen_stats.c
+++ b/net/core/gen_stats.c
@@ -106,16 +106,21 @@ gnet_stats_start_copy(struct sk_buff *skb, int type, spinlock_t *lock,
* if the room in the socket buffer was not sufficient.
*/
int
-gnet_stats_copy_basic(struct gnet_dump *d, struct gnet_stats_basic *b)
+gnet_stats_copy_basic(struct gnet_dump *d, struct gnet_stats_basic_packed *b)
{
if (d->compat_tc_stats) {
d->tc_stats.bytes = b->bytes;
d->tc_stats.packets = b->packets;
}
- if (d->tail)
- return gnet_stats_copy(d, TCA_STATS_BASIC, b, sizeof(*b));
+ if (d->tail) {
+ struct gnet_stats_basic sb;
+ memset(&sb, 0, sizeof(sb));
+ sb.bytes = b->bytes;
+ sb.packets = b->packets;
+ return gnet_stats_copy(d, TCA_STATS_BASIC, &sb, sizeof(sb));
+ }
return 0;
}
diff --git a/net/core/net_namespace.c b/net/core/net_namespace.c
index b7292a2719d..197283072cc 100644
--- a/net/core/net_namespace.c
+++ b/net/core/net_namespace.c
@@ -488,7 +488,7 @@ int net_assign_generic(struct net *net, int id, void *data)
*/
ng->len = id;
- memcpy(&ng->ptr, &old_ng->ptr, old_ng->len);
+ memcpy(&ng->ptr, &old_ng->ptr, old_ng->len * sizeof(void*));
rcu_assign_pointer(net->gen, ng);
call_rcu(&old_ng->rcu, net_generic_release);
diff --git a/net/core/netpoll.c b/net/core/netpoll.c
index 9675f312830..1b76eb11deb 100644
--- a/net/core/netpoll.c
+++ b/net/core/netpoll.c
@@ -319,6 +319,11 @@ static void netpoll_send_skb(struct netpoll *np, struct sk_buff *skb)
udelay(USEC_PER_POLL);
}
+
+ WARN_ONCE(!irqs_disabled(),
+ "netpoll_send_skb(): %s enabled interrupts in poll (%pF)\n",
+ dev->name, ops->ndo_start_xmit);
+
local_irq_restore(flags);
}
@@ -740,7 +745,7 @@ int netpoll_setup(struct netpoll *np)
np->name);
break;
}
- cond_resched();
+ msleep(1);
}
/* If carrier appears to come up instantly, we don't
diff --git a/net/core/skbuff.c b/net/core/skbuff.c
index 5c93435b034..9e0597d189b 100644
--- a/net/core/skbuff.c
+++ b/net/core/skbuff.c
@@ -204,6 +204,10 @@ struct sk_buff *__alloc_skb(unsigned int size, gfp_t gfp_mask,
skb->end = skb->tail + size;
kmemcheck_annotate_bitfield(skb, flags1);
kmemcheck_annotate_bitfield(skb, flags2);
+#ifdef NET_SKBUFF_DATA_USES_OFFSET
+ skb->mac_header = ~0U;
+#endif
+
/* make sure we initialize shinfo sequentially */
shinfo = skb_shinfo(skb);
atomic_set(&shinfo->dataref, 1);
@@ -665,7 +669,8 @@ static void copy_skb_header(struct sk_buff *new, const struct sk_buff *old)
/* {transport,network,mac}_header are relative to skb->head */
new->transport_header += offset;
new->network_header += offset;
- new->mac_header += offset;
+ if (skb_mac_header_was_set(new))
+ new->mac_header += offset;
#endif
skb_shinfo(new)->gso_size = skb_shinfo(old)->gso_size;
skb_shinfo(new)->gso_segs = skb_shinfo(old)->gso_segs;
@@ -847,7 +852,8 @@ int pskb_expand_head(struct sk_buff *skb, int nhead, int ntail,
skb->tail += off;
skb->transport_header += off;
skb->network_header += off;
- skb->mac_header += off;
+ if (skb_mac_header_was_set(skb))
+ skb->mac_header += off;
skb->csum_start += nhead;
skb->cloned = 0;
skb->hdr_len = 0;
@@ -939,7 +945,8 @@ struct sk_buff *skb_copy_expand(const struct sk_buff *skb,
#ifdef NET_SKBUFF_DATA_USES_OFFSET
n->transport_header += off;
n->network_header += off;
- n->mac_header += off;
+ if (skb_mac_header_was_set(skb))
+ n->mac_header += off;
#endif
return n;
diff --git a/net/core/sock.c b/net/core/sock.c
index b0ba569bc97..76334228ed1 100644
--- a/net/core/sock.c
+++ b/net/core/sock.c
@@ -631,7 +631,7 @@ set_rcvbuf:
case SO_TIMESTAMPING:
if (val & ~SOF_TIMESTAMPING_MASK) {
- ret = EINVAL;
+ ret = -EINVAL;
break;
}
sock_valbool_flag(sk, SOCK_TIMESTAMPING_TX_HARDWARE,
@@ -919,13 +919,19 @@ static inline void sock_lock_init(struct sock *sk)
af_family_keys + sk->sk_family);
}
+/*
+ * Copy all fields from osk to nsk but nsk->sk_refcnt must not change yet,
+ * even temporarly, because of RCU lookups. sk_node should also be left as is.
+ */
static void sock_copy(struct sock *nsk, const struct sock *osk)
{
#ifdef CONFIG_SECURITY_NETWORK
void *sptr = nsk->sk_security;
#endif
-
- memcpy(nsk, osk, osk->sk_prot->obj_size);
+ BUILD_BUG_ON(offsetof(struct sock, sk_copy_start) !=
+ sizeof(osk->sk_node) + sizeof(osk->sk_refcnt));
+ memcpy(&nsk->sk_copy_start, &osk->sk_copy_start,
+ osk->sk_prot->obj_size - offsetof(struct sock, sk_copy_start));
#ifdef CONFIG_SECURITY_NETWORK
nsk->sk_security = sptr;
security_sk_clone(osk, nsk);
@@ -939,8 +945,23 @@ static struct sock *sk_prot_alloc(struct proto *prot, gfp_t priority,
struct kmem_cache *slab;
slab = prot->slab;
- if (slab != NULL)
- sk = kmem_cache_alloc(slab, priority);
+ if (slab != NULL) {
+ sk = kmem_cache_alloc(slab, priority & ~__GFP_ZERO);
+ if (!sk)
+ return sk;
+ if (priority & __GFP_ZERO) {
+ /*
+ * caches using SLAB_DESTROY_BY_RCU should let
+ * sk_node.next un-modified. Special care is taken
+ * when initializing object to zero.
+ */
+ if (offsetof(struct sock, sk_node.next) != 0)
+ memset(sk, 0, offsetof(struct sock, sk_node.next));
+ memset(&sk->sk_node.pprev, 0,
+ prot->obj_size - offsetof(struct sock,
+ sk_node.pprev));
+ }
+ }
else
sk = kmalloc(prot->obj_size, priority);
@@ -1004,6 +1025,7 @@ struct sock *sk_alloc(struct net *net, int family, gfp_t priority,
sk->sk_prot = sk->sk_prot_creator = prot;
sock_lock_init(sk);
sock_net_set(sk, get_net(net));
+ atomic_set(&sk->sk_wmem_alloc, 1);
}
return sk;
@@ -1125,6 +1147,11 @@ struct sock *sk_clone(const struct sock *sk, const gfp_t priority)
newsk->sk_err = 0;
newsk->sk_priority = 0;
+ /*
+ * Before updating sk_refcnt, we must commit prior changes to memory
+ * (Documentation/RCU/rculist_nulls.txt for details)
+ */
+ smp_wmb();
atomic_set(&newsk->sk_refcnt, 2);
/*
@@ -1715,7 +1742,7 @@ EXPORT_SYMBOL(sock_no_sendpage);
static void sock_def_wakeup(struct sock *sk)
{
read_lock(&sk->sk_callback_lock);
- if (sk->sk_sleep && waitqueue_active(sk->sk_sleep))
+ if (sk_has_sleeper(sk))
wake_up_interruptible_all(sk->sk_sleep);
read_unlock(&sk->sk_callback_lock);
}
@@ -1723,7 +1750,7 @@ static void sock_def_wakeup(struct sock *sk)
static void sock_def_error_report(struct sock *sk)
{
read_lock(&sk->sk_callback_lock);
- if (sk->sk_sleep && waitqueue_active(sk->sk_sleep))
+ if (sk_has_sleeper(sk))
wake_up_interruptible_poll(sk->sk_sleep, POLLERR);
sk_wake_async(sk, SOCK_WAKE_IO, POLL_ERR);
read_unlock(&sk->sk_callback_lock);
@@ -1732,7 +1759,7 @@ static void sock_def_error_report(struct sock *sk)
static void sock_def_readable(struct sock *sk, int len)
{
read_lock(&sk->sk_callback_lock);
- if (sk->sk_sleep && waitqueue_active(sk->sk_sleep))
+ if (sk_has_sleeper(sk))
wake_up_interruptible_sync_poll(sk->sk_sleep, POLLIN |
POLLRDNORM | POLLRDBAND);
sk_wake_async(sk, SOCK_WAKE_WAITD, POLL_IN);
@@ -1747,7 +1774,7 @@ static void sock_def_write_space(struct sock *sk)
* progress. --DaveM
*/
if ((atomic_read(&sk->sk_wmem_alloc) << 1) <= sk->sk_sndbuf) {
- if (sk->sk_sleep && waitqueue_active(sk->sk_sleep))
+ if (sk_has_sleeper(sk))
wake_up_interruptible_sync_poll(sk->sk_sleep, POLLOUT |
POLLWRNORM | POLLWRBAND);
@@ -1840,8 +1867,12 @@ void sock_init_data(struct socket *sock, struct sock *sk)
sk->sk_stamp = ktime_set(-1L, 0);
+ /*
+ * Before updating sk_refcnt, we must commit prior changes to memory
+ * (Documentation/RCU/rculist_nulls.txt for details)
+ */
+ smp_wmb();
atomic_set(&sk->sk_refcnt, 1);
- atomic_set(&sk->sk_wmem_alloc, 1);
atomic_set(&sk->sk_drops, 0);
}
EXPORT_SYMBOL(sock_init_data);
generated by cgit v1.2.3-70-g09d2 (git 2.46.0) at 2025-03-05 22:02:12 +0000