summaryrefslogtreecommitdiffstats
path: root/drivers/net/macvtap.c
diff options
context:
space:
mode:
Diffstat (limited to 'drivers/net/macvtap.c')
-rw-r--r--drivers/net/macvtap.c410
1 files changed, 298 insertions, 112 deletions
diff --git a/drivers/net/macvtap.c b/drivers/net/macvtap.c
index b6dd6a75919..a98fb0ed6ae 100644
--- a/drivers/net/macvtap.c
+++ b/drivers/net/macvtap.c
@@ -31,10 +31,6 @@
* macvtap_proto is used to allocate queues through the sock allocation
* mechanism.
*
- * TODO: multiqueue support is currently not implemented, even though
- * macvtap is basically prepared for that. We will need to add this
- * here as well as in virtio-net and qemu to get line rate on 10gbit
- * adapters from a guest.
*/
struct macvtap_queue {
struct sock sk;
@@ -44,6 +40,9 @@ struct macvtap_queue {
struct macvlan_dev __rcu *vlan;
struct file *file;
unsigned int flags;
+ u16 queue_index;
+ bool enabled;
+ struct list_head next;
};
static struct proto macvtap_proto = {
@@ -66,11 +65,14 @@ static struct cdev macvtap_cdev;
static const struct proto_ops macvtap_socket_ops;
+#define TUN_OFFLOADS (NETIF_F_HW_CSUM | NETIF_F_TSO_ECN | NETIF_F_TSO | \
+ NETIF_F_TSO6 | NETIF_F_UFO)
+#define RX_OFFLOADS (NETIF_F_GRO | NETIF_F_LRO)
/*
* RCU usage:
* The macvtap_queue and the macvlan_dev are loosely coupled, the
* pointers from one to the other can only be read while rcu_read_lock
- * or macvtap_lock is held.
+ * or rtnl is held.
*
* Both the file and the macvlan_dev hold a reference on the macvtap_queue
* through sock_hold(&q->sk). When the macvlan_dev goes away first,
@@ -82,54 +84,84 @@ static const struct proto_ops macvtap_socket_ops;
* file or the dev. The data structure is freed through __sk_free
* when both our references and any pending SKBs are gone.
*/
-static DEFINE_SPINLOCK(macvtap_lock);
-/*
- * get_slot: return a [unused/occupied] slot in vlan->taps[]:
- * - if 'q' is NULL, return the first empty slot;
- * - otherwise, return the slot this pointer occupies.
- */
-static int get_slot(struct macvlan_dev *vlan, struct macvtap_queue *q)
+static int macvtap_enable_queue(struct net_device *dev, struct file *file,
+ struct macvtap_queue *q)
{
- int i;
+ struct macvlan_dev *vlan = netdev_priv(dev);
+ int err = -EINVAL;
- for (i = 0; i < MAX_MACVTAP_QUEUES; i++) {
- if (rcu_dereference_protected(vlan->taps[i],
- lockdep_is_held(&macvtap_lock)) == q)
- return i;
- }
+ ASSERT_RTNL();
+
+ if (q->enabled)
+ goto out;
+
+ err = 0;
+ rcu_assign_pointer(vlan->taps[vlan->numvtaps], q);
+ q->queue_index = vlan->numvtaps;
+ q->enabled = true;
- /* Should never happen */
- BUG_ON(1);
+ vlan->numvtaps++;
+out:
+ return err;
}
static int macvtap_set_queue(struct net_device *dev, struct file *file,
- struct macvtap_queue *q)
+ struct macvtap_queue *q)
{
struct macvlan_dev *vlan = netdev_priv(dev);
- int index;
int err = -EBUSY;
- spin_lock(&macvtap_lock);
- if (vlan->numvtaps == MAX_MACVTAP_QUEUES)
+ rtnl_lock();
+ if (vlan->numqueues == MAX_MACVTAP_QUEUES)
goto out;
err = 0;
- index = get_slot(vlan, NULL);
rcu_assign_pointer(q->vlan, vlan);
- rcu_assign_pointer(vlan->taps[index], q);
+ rcu_assign_pointer(vlan->taps[vlan->numvtaps], q);
sock_hold(&q->sk);
q->file = file;
+ q->queue_index = vlan->numvtaps;
+ q->enabled = true;
file->private_data = q;
+ list_add_tail(&q->next, &vlan->queue_list);
vlan->numvtaps++;
+ vlan->numqueues++;
out:
- spin_unlock(&macvtap_lock);
+ rtnl_unlock();
return err;
}
+static int macvtap_disable_queue(struct macvtap_queue *q)
+{
+ struct macvlan_dev *vlan;
+ struct macvtap_queue *nq;
+
+ ASSERT_RTNL();
+ if (!q->enabled)
+ return -EINVAL;
+
+ vlan = rtnl_dereference(q->vlan);
+
+ if (vlan) {
+ int index = q->queue_index;
+ BUG_ON(index >= vlan->numvtaps);
+ nq = rtnl_dereference(vlan->taps[vlan->numvtaps - 1]);
+ nq->queue_index = index;
+
+ rcu_assign_pointer(vlan->taps[index], nq);
+ RCU_INIT_POINTER(vlan->taps[vlan->numvtaps - 1], NULL);
+ q->enabled = false;
+
+ vlan->numvtaps--;
+ }
+
+ return 0;
+}
+
/*
* The file owning the queue got closed, give up both
* the reference that the files holds as well as the
@@ -142,19 +174,20 @@ static void macvtap_put_queue(struct macvtap_queue *q)
{
struct macvlan_dev *vlan;
- spin_lock(&macvtap_lock);
- vlan = rcu_dereference_protected(q->vlan,
- lockdep_is_held(&macvtap_lock));
+ rtnl_lock();
+ vlan = rtnl_dereference(q->vlan);
+
if (vlan) {
- int index = get_slot(vlan, q);
+ if (q->enabled)
+ BUG_ON(macvtap_disable_queue(q));
- RCU_INIT_POINTER(vlan->taps[index], NULL);
+ vlan->numqueues--;
RCU_INIT_POINTER(q->vlan, NULL);
sock_put(&q->sk);
- --vlan->numvtaps;
+ list_del_init(&q->next);
}
- spin_unlock(&macvtap_lock);
+ rtnl_unlock();
synchronize_rcu();
sock_put(&q->sk);
@@ -172,7 +205,12 @@ static struct macvtap_queue *macvtap_get_queue(struct net_device *dev,
{
struct macvlan_dev *vlan = netdev_priv(dev);
struct macvtap_queue *tap = NULL;
- int numvtaps = vlan->numvtaps;
+ /* Access to taps array is protected by rcu, but access to numvtaps
+ * isn't. Below we use it to lookup a queue, but treat it as a hint
+ * and validate that the result isn't NULL - in case we are
+ * racing against queue removal.
+ */
+ int numvtaps = ACCESS_ONCE(vlan->numvtaps);
__u32 rxq;
if (!numvtaps)
@@ -182,8 +220,7 @@ static struct macvtap_queue *macvtap_get_queue(struct net_device *dev,
rxq = skb_get_rxhash(skb);
if (rxq) {
tap = rcu_dereference(vlan->taps[rxq % numvtaps]);
- if (tap)
- goto out;
+ goto out;
}
if (likely(skb_rx_queue_recorded(skb))) {
@@ -193,17 +230,10 @@ static struct macvtap_queue *macvtap_get_queue(struct net_device *dev,
rxq -= numvtaps;
tap = rcu_dereference(vlan->taps[rxq]);
- if (tap)
- goto out;
- }
-
- /* Everything failed - find first available queue */
- for (rxq = 0; rxq < MAX_MACVTAP_QUEUES; rxq++) {
- tap = rcu_dereference(vlan->taps[rxq]);
- if (tap)
- break;
+ goto out;
}
+ tap = rcu_dereference(vlan->taps[0]);
out:
return tap;
}
@@ -216,27 +246,24 @@ out:
static void macvtap_del_queues(struct net_device *dev)
{
struct macvlan_dev *vlan = netdev_priv(dev);
- struct macvtap_queue *q, *qlist[MAX_MACVTAP_QUEUES];
+ struct macvtap_queue *q, *tmp, *qlist[MAX_MACVTAP_QUEUES];
int i, j = 0;
- /* macvtap_put_queue can free some slots, so go through all slots */
- spin_lock(&macvtap_lock);
- for (i = 0; i < MAX_MACVTAP_QUEUES && vlan->numvtaps; i++) {
- q = rcu_dereference_protected(vlan->taps[i],
- lockdep_is_held(&macvtap_lock));
- if (q) {
- qlist[j++] = q;
- RCU_INIT_POINTER(vlan->taps[i], NULL);
- RCU_INIT_POINTER(q->vlan, NULL);
+ ASSERT_RTNL();
+ list_for_each_entry_safe(q, tmp, &vlan->queue_list, next) {
+ list_del_init(&q->next);
+ qlist[j++] = q;
+ RCU_INIT_POINTER(q->vlan, NULL);
+ if (q->enabled)
vlan->numvtaps--;
- }
+ vlan->numqueues--;
}
- BUG_ON(vlan->numvtaps != 0);
+ for (i = 0; i < vlan->numvtaps; i++)
+ RCU_INIT_POINTER(vlan->taps[i], NULL);
+ BUG_ON(vlan->numvtaps);
+ BUG_ON(vlan->numqueues);
/* guarantee that any future macvtap_set_queue will fail */
vlan->numvtaps = MAX_MACVTAP_QUEUES;
- spin_unlock(&macvtap_lock);
-
- synchronize_rcu();
for (--j; j >= 0; j--)
sock_put(&qlist[j]->sk);
@@ -249,14 +276,44 @@ static void macvtap_del_queues(struct net_device *dev)
*/
static int macvtap_forward(struct net_device *dev, struct sk_buff *skb)
{
+ struct macvlan_dev *vlan = netdev_priv(dev);
struct macvtap_queue *q = macvtap_get_queue(dev, skb);
+ netdev_features_t features;
if (!q)
goto drop;
if (skb_queue_len(&q->sk.sk_receive_queue) >= dev->tx_queue_len)
goto drop;
- skb_queue_tail(&q->sk.sk_receive_queue, skb);
+ skb->dev = dev;
+ /* Apply the forward feature mask so that we perform segmentation
+ * according to users wishes.
+ */
+ features = netif_skb_features(skb) & vlan->tap_features;
+ if (netif_needs_gso(skb, features)) {
+ struct sk_buff *segs = __skb_gso_segment(skb, features, false);
+
+ if (IS_ERR(segs))
+ goto drop;
+
+ if (!segs) {
+ skb_queue_tail(&q->sk.sk_receive_queue, skb);
+ goto wake_up;
+ }
+
+ kfree_skb(skb);
+ while (segs) {
+ struct sk_buff *nskb = segs->next;
+
+ segs->next = NULL;
+ skb_queue_tail(&q->sk.sk_receive_queue, segs);
+ segs = nskb;
+ }
+ } else {
+ skb_queue_tail(&q->sk.sk_receive_queue, skb);
+ }
+
+wake_up:
wake_up_interruptible_poll(sk_sleep(&q->sk), POLLIN | POLLRDNORM | POLLRDBAND);
return NET_RX_SUCCESS;
@@ -322,6 +379,14 @@ static int macvtap_newlink(struct net *src_net,
struct nlattr *tb[],
struct nlattr *data[])
{
+ struct macvlan_dev *vlan = netdev_priv(dev);
+ INIT_LIST_HEAD(&vlan->queue_list);
+
+ /* Since macvlan supports all offloads by default, make
+ * tap support all offloads also.
+ */
+ vlan->tap_features = TUN_OFFLOADS;
+
/* Don't put anything that may fail after macvlan_common_newlink
* because we can't undo what it does.
*/
@@ -385,7 +450,7 @@ static int macvtap_open(struct inode *inode, struct file *file)
if (!q)
goto out;
- q->sock.wq = &q->wq;
+ RCU_INIT_POINTER(q->sock.wq, &q->wq);
init_waitqueue_head(&q->wq.wait);
q->sock.type = SOCK_RAW;
q->sock.state = SS_CONNECTED;
@@ -633,6 +698,28 @@ static int macvtap_skb_to_vnet_hdr(const struct sk_buff *skb,
return 0;
}
+static unsigned long iov_pages(const struct iovec *iv, int offset,
+ unsigned long nr_segs)
+{
+ unsigned long seg, base;
+ int pages = 0, len, size;
+
+ while (nr_segs && (offset >= iv->iov_len)) {
+ offset -= iv->iov_len;
+ ++iv;
+ --nr_segs;
+ }
+
+ for (seg = 0; seg < nr_segs; seg++) {
+ base = (unsigned long)iv[seg].iov_base + offset;
+ len = iv[seg].iov_len - offset;
+ size = ((base & ~PAGE_MASK) + len + ~PAGE_MASK) >> PAGE_SHIFT;
+ pages += size;
+ offset = 0;
+ }
+
+ return pages;
+}
/* Get packet from user space buffer */
static ssize_t macvtap_get_user(struct macvtap_queue *q, struct msghdr *m,
@@ -647,6 +734,7 @@ static ssize_t macvtap_get_user(struct macvtap_queue *q, struct msghdr *m,
int vnet_hdr_len = 0;
int copylen = 0;
bool zerocopy = false;
+ size_t linear;
if (q->flags & IFF_VNET_HDR) {
vnet_hdr_len = q->vnet_hdr_sz;
@@ -678,42 +766,35 @@ static ssize_t macvtap_get_user(struct macvtap_queue *q, struct msghdr *m,
if (unlikely(count > UIO_MAXIOV))
goto err;
- if (m && m->msg_control && sock_flag(&q->sk, SOCK_ZEROCOPY))
- zerocopy = true;
+ if (m && m->msg_control && sock_flag(&q->sk, SOCK_ZEROCOPY)) {
+ copylen = vnet_hdr.hdr_len ? vnet_hdr.hdr_len : GOODCOPY_LEN;
+ linear = copylen;
+ if (iov_pages(iv, vnet_hdr_len + copylen, count)
+ <= MAX_SKB_FRAGS)
+ zerocopy = true;
+ }
- if (zerocopy) {
- /* Userspace may produce vectors with count greater than
- * MAX_SKB_FRAGS, so we need to linearize parts of the skb
- * to let the rest of data to be fit in the frags.
- */
- if (count > MAX_SKB_FRAGS) {
- copylen = iov_length(iv, count - MAX_SKB_FRAGS);
- if (copylen < vnet_hdr_len)
- copylen = 0;
- else
- copylen -= vnet_hdr_len;
- }
- /* There are 256 bytes to be copied in skb, so there is enough
- * room for skb expand head in case it is used.
- * The rest buffer is mapped from userspace.
- */
- if (copylen < vnet_hdr.hdr_len)
- copylen = vnet_hdr.hdr_len;
- if (!copylen)
- copylen = GOODCOPY_LEN;
- } else
+ if (!zerocopy) {
copylen = len;
+ linear = vnet_hdr.hdr_len;
+ }
skb = macvtap_alloc_skb(&q->sk, NET_IP_ALIGN, copylen,
- vnet_hdr.hdr_len, noblock, &err);
+ linear, noblock, &err);
if (!skb)
goto err;
if (zerocopy)
err = zerocopy_sg_from_iovec(skb, iv, vnet_hdr_len, count);
- else
+ else {
err = skb_copy_datagram_from_iovec(skb, 0, iv, vnet_hdr_len,
len);
+ if (!err && m && m->msg_control) {
+ struct ubuf_info *uarg = m->msg_control;
+ uarg->callback(uarg, false);
+ }
+ }
+
if (err)
goto err_kfree;
@@ -729,8 +810,8 @@ static ssize_t macvtap_get_user(struct macvtap_queue *q, struct msghdr *m,
skb_probe_transport_header(skb, ETH_HLEN);
- rcu_read_lock_bh();
- vlan = rcu_dereference_bh(q->vlan);
+ rcu_read_lock();
+ vlan = rcu_dereference(q->vlan);
/* copy skb_ubuf_info for callback when skb has no error */
if (zerocopy) {
skb_shinfo(skb)->destructor_arg = m->msg_control;
@@ -741,7 +822,7 @@ static ssize_t macvtap_get_user(struct macvtap_queue *q, struct msghdr *m,
macvlan_start_xmit(skb, vlan->dev);
else
kfree_skb(skb);
- rcu_read_unlock_bh();
+ rcu_read_unlock();
return total_len;
@@ -749,11 +830,11 @@ err_kfree:
kfree_skb(skb);
err:
- rcu_read_lock_bh();
- vlan = rcu_dereference_bh(q->vlan);
+ rcu_read_lock();
+ vlan = rcu_dereference(q->vlan);
if (vlan)
vlan->dev->stats.tx_dropped++;
- rcu_read_unlock_bh();
+ rcu_read_unlock();
return err;
}
@@ -804,7 +885,7 @@ static ssize_t macvtap_put_user(struct macvtap_queue *q,
__be16 h_vlan_proto;
__be16 h_vlan_TCI;
} veth;
- veth.h_vlan_proto = htons(ETH_P_8021Q);
+ veth.h_vlan_proto = skb->vlan_proto;
veth.h_vlan_TCI = htons(vlan_tx_tag_get(skb));
vlan_offset = offsetof(struct vlan_ethhdr, h_vlan_proto);
@@ -829,11 +910,11 @@ static ssize_t macvtap_put_user(struct macvtap_queue *q,
copied += len;
done:
- rcu_read_lock_bh();
- vlan = rcu_dereference_bh(q->vlan);
+ rcu_read_lock();
+ vlan = rcu_dereference(q->vlan);
if (vlan)
macvlan_count_rx(vlan, copied - vnet_hdr_len, ret == 0, 0);
- rcu_read_unlock_bh();
+ rcu_read_unlock();
return ret ? ret : copied;
}
@@ -847,7 +928,9 @@ static ssize_t macvtap_do_read(struct macvtap_queue *q, struct kiocb *iocb,
ssize_t ret = 0;
while (len) {
- prepare_to_wait(sk_sleep(&q->sk), &wait, TASK_INTERRUPTIBLE);
+ if (!noblock)
+ prepare_to_wait(sk_sleep(&q->sk), &wait,
+ TASK_INTERRUPTIBLE);
/* Read frames from the queue */
skb = skb_dequeue(&q->sk.sk_receive_queue);
@@ -869,7 +952,8 @@ static ssize_t macvtap_do_read(struct macvtap_queue *q, struct kiocb *iocb,
break;
}
- finish_wait(sk_sleep(&q->sk), &wait);
+ if (!noblock)
+ finish_wait(sk_sleep(&q->sk), &wait);
return ret;
}
@@ -892,6 +976,96 @@ out:
return ret;
}
+static struct macvlan_dev *macvtap_get_vlan(struct macvtap_queue *q)
+{
+ struct macvlan_dev *vlan;
+
+ ASSERT_RTNL();
+ vlan = rtnl_dereference(q->vlan);
+ if (vlan)
+ dev_hold(vlan->dev);
+
+ return vlan;
+}
+
+static void macvtap_put_vlan(struct macvlan_dev *vlan)
+{
+ dev_put(vlan->dev);
+}
+
+static int macvtap_ioctl_set_queue(struct file *file, unsigned int flags)
+{
+ struct macvtap_queue *q = file->private_data;
+ struct macvlan_dev *vlan;
+ int ret;
+
+ vlan = macvtap_get_vlan(q);
+ if (!vlan)
+ return -EINVAL;
+
+ if (flags & IFF_ATTACH_QUEUE)
+ ret = macvtap_enable_queue(vlan->dev, file, q);
+ else if (flags & IFF_DETACH_QUEUE)
+ ret = macvtap_disable_queue(q);
+ else
+ ret = -EINVAL;
+
+ macvtap_put_vlan(vlan);
+ return ret;
+}
+
+static int set_offload(struct macvtap_queue *q, unsigned long arg)
+{
+ struct macvlan_dev *vlan;
+ netdev_features_t features;
+ netdev_features_t feature_mask = 0;
+
+ vlan = rtnl_dereference(q->vlan);
+ if (!vlan)
+ return -ENOLINK;
+
+ features = vlan->dev->features;
+
+ if (arg & TUN_F_CSUM) {
+ feature_mask = NETIF_F_HW_CSUM;
+
+ if (arg & (TUN_F_TSO4 | TUN_F_TSO6)) {
+ if (arg & TUN_F_TSO_ECN)
+ feature_mask |= NETIF_F_TSO_ECN;
+ if (arg & TUN_F_TSO4)
+ feature_mask |= NETIF_F_TSO;
+ if (arg & TUN_F_TSO6)
+ feature_mask |= NETIF_F_TSO6;
+ }
+
+ if (arg & TUN_F_UFO)
+ feature_mask |= NETIF_F_UFO;
+ }
+
+ /* tun/tap driver inverts the usage for TSO offloads, where
+ * setting the TSO bit means that the userspace wants to
+ * accept TSO frames and turning it off means that user space
+ * does not support TSO.
+ * For macvtap, we have to invert it to mean the same thing.
+ * When user space turns off TSO, we turn off GSO/LRO so that
+ * user-space will not receive TSO frames.
+ */
+ if (feature_mask & (NETIF_F_TSO | NETIF_F_TSO6 | NETIF_F_UFO))
+ features |= RX_OFFLOADS;
+ else
+ features &= ~RX_OFFLOADS;
+
+ /* tap_features are the same as features on tun/tap and
+ * reflect user expectations.
+ */
+ vlan->tap_features = vlan->dev->features &
+ (feature_mask | ~TUN_OFFLOADS);
+ vlan->set_features = features;
+ netdev_update_features(vlan->dev);
+
+ return 0;
+}
+
/*
* provide compatibility with generic tun/tap interface
*/
@@ -915,7 +1089,8 @@ static long macvtap_ioctl(struct file *file, unsigned int cmd,
return -EFAULT;
ret = 0;
- if ((u & ~IFF_VNET_HDR) != (IFF_NO_PI | IFF_TAP))
+ if ((u & ~(IFF_VNET_HDR | IFF_MULTI_QUEUE)) !=
+ (IFF_NO_PI | IFF_TAP))
ret = -EINVAL;
else
q->flags = u;
@@ -923,24 +1098,32 @@ static long macvtap_ioctl(struct file *file, unsigned int cmd,
return ret;
case TUNGETIFF:
- rcu_read_lock_bh();
- vlan = rcu_dereference_bh(q->vlan);
- if (vlan)
- dev_hold(vlan->dev);
- rcu_read_unlock_bh();
-
- if (!vlan)
+ rtnl_lock();
+ vlan = macvtap_get_vlan(q);
+ if (!vlan) {
+ rtnl_unlock();
return -ENOLINK;
+ }
ret = 0;
if (copy_to_user(&ifr->ifr_name, vlan->dev->name, IFNAMSIZ) ||
put_user(q->flags, &ifr->ifr_flags))
ret = -EFAULT;
- dev_put(vlan->dev);
+ macvtap_put_vlan(vlan);
+ rtnl_unlock();
+ return ret;
+
+ case TUNSETQUEUE:
+ if (get_user(u, &ifr->ifr_flags))
+ return -EFAULT;
+ rtnl_lock();
+ ret = macvtap_ioctl_set_queue(file, u);
+ rtnl_unlock();
return ret;
case TUNGETFEATURES:
- if (put_user(IFF_TAP | IFF_NO_PI | IFF_VNET_HDR, up))
+ if (put_user(IFF_TAP | IFF_NO_PI | IFF_VNET_HDR |
+ IFF_MULTI_QUEUE, up))
return -EFAULT;
return 0;
@@ -976,7 +1159,10 @@ static long macvtap_ioctl(struct file *file, unsigned int cmd,
got enabled for forwarded frames */
if (!(q->flags & IFF_VNET_HDR))
return -EINVAL;
- return 0;
+ rtnl_lock();
+ ret = set_offload(q, arg);
+ rtnl_unlock();
+ return ret;
default:
return -EINVAL;
@@ -1055,7 +1241,7 @@ EXPORT_SYMBOL_GPL(macvtap_get_socket);
static int macvtap_device_event(struct notifier_block *unused,
unsigned long event, void *ptr)
{
- struct net_device *dev = ptr;
+ struct net_device *dev = netdev_notifier_info_to_dev(ptr);
struct macvlan_dev *vlan;
struct device *classdev;
dev_t devt;