Merge branch 'master' of git://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux-2.6

Conflicts:
	drivers/power/wm97xx_battery.c

1 files changed, 624 insertions, 454 deletions

diff --git a/drivers/scsi/fcoe/fcoe.c b/drivers/scsi/fcoe/fcoe.c
index 0a5609bb581..704b8e03494 100644
--- a/drivers/scsi/fcoe/fcoe.c
+++ b/drivers/scsi/fcoe/fcoe.c
@@ -1,5 +1,5 @@
 /*
- * Copyright(c) 2007 - 2008 Intel Corporation. All rights reserved.
+ * Copyright(c) 2007 - 2009 Intel Corporation. All rights reserved.
  *
  * This program is free software; you can redistribute it and/or modify it
  * under the terms and conditions of the GNU General Public License,
@@ -49,9 +49,20 @@ MODULE_AUTHOR("Open-FCoE.org");
 MODULE_DESCRIPTION("FCoE");
 MODULE_LICENSE("GPL v2");
 
+/* Performance tuning parameters for fcoe */
+static unsigned int fcoe_ddp_min;
+module_param_named(ddp_min, fcoe_ddp_min, uint, S_IRUGO | S_IWUSR);
+MODULE_PARM_DESC(ddp_min, "Minimum I/O size in bytes for "	\
+		 "Direct Data Placement (DDP).");
+
+DEFINE_MUTEX(fcoe_config_mutex);
+
+/* fcoe_percpu_clean completion.  Waiter protected by fcoe_create_mutex */
+static DECLARE_COMPLETION(fcoe_flush_completion);
+
 /* fcoe host list */
+/* must only by accessed under the RTNL mutex */
 LIST_HEAD(fcoe_hostlist);
-DEFINE_RWLOCK(fcoe_hostlist_lock);
 DEFINE_PER_CPU(struct fcoe_percpu_s, fcoe_percpu);
 
 /* Function Prototypes */
@@ -66,12 +77,13 @@ static int fcoe_link_ok(struct fc_lport *lp);
 
 static struct fc_lport *fcoe_hostlist_lookup(const struct net_device *);
 static int fcoe_hostlist_add(const struct fc_lport *);
-static int fcoe_hostlist_remove(const struct fc_lport *);
 
 static void fcoe_check_wait_queue(struct fc_lport *, struct sk_buff *);
 static int fcoe_device_notification(struct notifier_block *, ulong, void *);
 static void fcoe_dev_setup(void);
 static void fcoe_dev_cleanup(void);
+static struct fcoe_interface *
+	fcoe_hostlist_lookup_port(const struct net_device *dev);
 
 /* notification function from net device */
 static struct notifier_block fcoe_notifier = {
@@ -132,6 +144,180 @@ static struct scsi_host_template fcoe_shost_template = {
 	.max_sectors = 0xffff,
 };
 
+static int fcoe_fip_recv(struct sk_buff *skb, struct net_device *dev,
+			 struct packet_type *ptype,
+			 struct net_device *orig_dev);
+/**
+ * fcoe_interface_setup()
+ * @fcoe: new fcoe_interface
+ * @netdev : ptr to the associated netdevice struct
+ *
+ * Returns : 0 for success
+ * Locking: must be called with the RTNL mutex held
+ */
+static int fcoe_interface_setup(struct fcoe_interface *fcoe,
+				struct net_device *netdev)
+{
+	struct fcoe_ctlr *fip = &fcoe->ctlr;
+	struct netdev_hw_addr *ha;
+	u8 flogi_maddr[ETH_ALEN];
+
+	fcoe->netdev = netdev;
+
+	/* Do not support for bonding device */
+	if ((netdev->priv_flags & IFF_MASTER_ALB) ||
+	    (netdev->priv_flags & IFF_SLAVE_INACTIVE) ||
+	    (netdev->priv_flags & IFF_MASTER_8023AD)) {
+		return -EOPNOTSUPP;
+	}
+
+	/* look for SAN MAC address, if multiple SAN MACs exist, only
+	 * use the first one for SPMA */
+	rcu_read_lock();
+	for_each_dev_addr(netdev, ha) {
+		if ((ha->type == NETDEV_HW_ADDR_T_SAN) &&
+		    (is_valid_ether_addr(fip->ctl_src_addr))) {
+			memcpy(fip->ctl_src_addr, ha->addr, ETH_ALEN);
+			fip->spma = 1;
+			break;
+		}
+	}
+	rcu_read_unlock();
+
+	/* setup Source Mac Address */
+	if (!fip->spma)
+		memcpy(fip->ctl_src_addr, netdev->dev_addr, netdev->addr_len);
+
+	/*
+	 * Add FCoE MAC address as second unicast MAC address
+	 * or enter promiscuous mode if not capable of listening
+	 * for multiple unicast MACs.
+	 */
+	memcpy(flogi_maddr, (u8[6]) FC_FCOE_FLOGI_MAC, ETH_ALEN);
+	dev_unicast_add(netdev, flogi_maddr);
+	if (fip->spma)
+		dev_unicast_add(netdev, fip->ctl_src_addr);
+	dev_mc_add(netdev, FIP_ALL_ENODE_MACS, ETH_ALEN, 0);
+
+	/*
+	 * setup the receive function from ethernet driver
+	 * on the ethertype for the given device
+	 */
+	fcoe->fcoe_packet_type.func = fcoe_rcv;
+	fcoe->fcoe_packet_type.type = __constant_htons(ETH_P_FCOE);
+	fcoe->fcoe_packet_type.dev = netdev;
+	dev_add_pack(&fcoe->fcoe_packet_type);
+
+	fcoe->fip_packet_type.func = fcoe_fip_recv;
+	fcoe->fip_packet_type.type = htons(ETH_P_FIP);
+	fcoe->fip_packet_type.dev = netdev;
+	dev_add_pack(&fcoe->fip_packet_type);
+
+	return 0;
+}
+
+static void fcoe_fip_send(struct fcoe_ctlr *fip, struct sk_buff *skb);
+static void fcoe_update_src_mac(struct fcoe_ctlr *fip, u8 *old, u8 *new);
+static void fcoe_destroy_work(struct work_struct *work);
+
+/**
+ * fcoe_interface_create()
+ * @netdev: network interface
+ *
+ * Returns: pointer to a struct fcoe_interface or NULL on error
+ */
+static struct fcoe_interface *fcoe_interface_create(struct net_device *netdev)
+{
+	struct fcoe_interface *fcoe;
+
+	fcoe = kzalloc(sizeof(*fcoe), GFP_KERNEL);
+	if (!fcoe) {
+		FCOE_NETDEV_DBG(netdev, "Could not allocate fcoe structure\n");
+		return NULL;
+	}
+
+	dev_hold(netdev);
+	kref_init(&fcoe->kref);
+
+	/*
+	 * Initialize FIP.
+	 */
+	fcoe_ctlr_init(&fcoe->ctlr);
+	fcoe->ctlr.send = fcoe_fip_send;
+	fcoe->ctlr.update_mac = fcoe_update_src_mac;
+
+	fcoe_interface_setup(fcoe, netdev);
+
+	return fcoe;
+}
+
+/**
+ * fcoe_interface_cleanup() - clean up netdev configurations
+ * @fcoe:
+ *
+ * Caller must be holding the RTNL mutex
+ */
+void fcoe_interface_cleanup(struct fcoe_interface *fcoe)
+{
+	struct net_device *netdev = fcoe->netdev;
+	struct fcoe_ctlr *fip = &fcoe->ctlr;
+	u8 flogi_maddr[ETH_ALEN];
+
+	/*
+	 * Don't listen for Ethernet packets anymore.
+	 * synchronize_net() ensures that the packet handlers are not running
+	 * on another CPU. dev_remove_pack() would do that, this calls the
+	 * unsyncronized version __dev_remove_pack() to avoid multiple delays.
+	 */
+	__dev_remove_pack(&fcoe->fcoe_packet_type);
+	__dev_remove_pack(&fcoe->fip_packet_type);
+	synchronize_net();
+
+	/* Delete secondary MAC addresses */
+	memcpy(flogi_maddr, (u8[6]) FC_FCOE_FLOGI_MAC, ETH_ALEN);
+	dev_unicast_delete(netdev, flogi_maddr);
+	if (!is_zero_ether_addr(fip->data_src_addr))
+		dev_unicast_delete(netdev, fip->data_src_addr);
+	if (fip->spma)
+		dev_unicast_delete(netdev, fip->ctl_src_addr);
+	dev_mc_delete(netdev, FIP_ALL_ENODE_MACS, ETH_ALEN, 0);
+}
+
+/**
+ * fcoe_interface_release() - fcoe_port kref release function
+ * @kref: embedded reference count in an fcoe_interface struct
+ */
+static void fcoe_interface_release(struct kref *kref)
+{
+	struct fcoe_interface *fcoe;
+	struct net_device *netdev;
+
+	fcoe = container_of(kref, struct fcoe_interface, kref);
+	netdev = fcoe->netdev;
+	/* tear-down the FCoE controller */
+	fcoe_ctlr_destroy(&fcoe->ctlr);
+	kfree(fcoe);
+	dev_put(netdev);
+}
+
+/**
+ * fcoe_interface_get()
+ * @fcoe:
+ */
+static inline void fcoe_interface_get(struct fcoe_interface *fcoe)
+{
+	kref_get(&fcoe->kref);
+}
+
+/**
+ * fcoe_interface_put()
+ * @fcoe:
+ */
+static inline void fcoe_interface_put(struct fcoe_interface *fcoe)
+{
+	kref_put(&fcoe->kref, fcoe_interface_release);
+}
+
 /**
  * fcoe_fip_recv - handle a received FIP frame.
  * @skb: the receive skb
@@ -145,10 +331,10 @@ static int fcoe_fip_recv(struct sk_buff *skb, struct net_device *dev,
 			 struct packet_type *ptype,
 			 struct net_device *orig_dev)
 {
-	struct fcoe_softc *fc;
+	struct fcoe_interface *fcoe;
 
-	fc = container_of(ptype, struct fcoe_softc, fip_packet_type);
-	fcoe_ctlr_recv(&fc->ctlr, skb);
+	fcoe = container_of(ptype, struct fcoe_interface, fip_packet_type);
+	fcoe_ctlr_recv(&fcoe->ctlr, skb);
 	return 0;
 }
 
@@ -159,7 +345,7 @@ static int fcoe_fip_recv(struct sk_buff *skb, struct net_device *dev,
  */
 static void fcoe_fip_send(struct fcoe_ctlr *fip, struct sk_buff *skb)
 {
-	skb->dev = fcoe_from_ctlr(fip)->real_dev;
+	skb->dev = fcoe_from_ctlr(fip)->netdev;
 	dev_queue_xmit(skb);
 }
 
@@ -174,13 +360,13 @@ static void fcoe_fip_send(struct fcoe_ctlr *fip, struct sk_buff *skb)
  */
 static void fcoe_update_src_mac(struct fcoe_ctlr *fip, u8 *old, u8 *new)
 {
-	struct fcoe_softc *fc;
+	struct fcoe_interface *fcoe;
 
-	fc = fcoe_from_ctlr(fip);
+	fcoe = fcoe_from_ctlr(fip);
 	rtnl_lock();
 	if (!is_zero_ether_addr(old))
-		dev_unicast_delete(fc->real_dev, old);
-	dev_unicast_add(fc->real_dev, new);
+		dev_unicast_delete(fcoe->netdev, old);
+	dev_unicast_add(fcoe->netdev, new);
 	rtnl_unlock();
 }
 
@@ -217,30 +403,6 @@ static int fcoe_lport_config(struct fc_lport *lp)
 }
 
 /**
- * fcoe_netdev_cleanup() - clean up netdev configurations
- * @fc: ptr to the fcoe_softc
- */
-void fcoe_netdev_cleanup(struct fcoe_softc *fc)
-{
-	u8 flogi_maddr[ETH_ALEN];
-
-	/* Don't listen for Ethernet packets anymore */
-	dev_remove_pack(&fc->fcoe_packet_type);
-	dev_remove_pack(&fc->fip_packet_type);
-
-	/* Delete secondary MAC addresses */
-	rtnl_lock();
-	memcpy(flogi_maddr, (u8[6]) FC_FCOE_FLOGI_MAC, ETH_ALEN);
-	dev_unicast_delete(fc->real_dev, flogi_maddr);
-	if (!is_zero_ether_addr(fc->ctlr.data_src_addr))
-		dev_unicast_delete(fc->real_dev, fc->ctlr.data_src_addr);
-	if (fc->ctlr.spma)
-		dev_unicast_delete(fc->real_dev, fc->ctlr.ctl_src_addr);
-	dev_mc_delete(fc->real_dev, FIP_ALL_ENODE_MACS, ETH_ALEN, 0);
-	rtnl_unlock();
-}
-
-/**
  * fcoe_queue_timer() - fcoe queue timer
  * @lp: the fc_lport pointer
  *
@@ -265,116 +427,53 @@ static int fcoe_netdev_config(struct fc_lport *lp, struct net_device *netdev)
 {
 	u32 mfs;
 	u64 wwnn, wwpn;
-	struct fcoe_softc *fc;
-	u8 flogi_maddr[ETH_ALEN];
-	struct netdev_hw_addr *ha;
+	struct fcoe_interface *fcoe;
+	struct fcoe_port *port;
 
 	/* Setup lport private data to point to fcoe softc */
-	fc = lport_priv(lp);
-	fc->ctlr.lp = lp;
-	fc->real_dev = netdev;
-	fc->phys_dev = netdev;
-
-	/* Require support for get_pauseparam ethtool op. */
-	if (netdev->priv_flags & IFF_802_1Q_VLAN)
-		fc->phys_dev = vlan_dev_real_dev(netdev);
-
-	/* Do not support for bonding device */
-	if ((fc->real_dev->priv_flags & IFF_MASTER_ALB) ||
-	    (fc->real_dev->priv_flags & IFF_SLAVE_INACTIVE) ||
-	    (fc->real_dev->priv_flags & IFF_MASTER_8023AD)) {
-		return -EOPNOTSUPP;
-	}
+	port = lport_priv(lp);
+	fcoe = port->fcoe;
 
 	/*
 	 * Determine max frame size based on underlying device and optional
 	 * user-configured limit.  If the MFS is too low, fcoe_link_ok()
 	 * will return 0, so do this first.
 	 */
-	mfs = fc->real_dev->mtu - (sizeof(struct fcoe_hdr) +
-				   sizeof(struct fcoe_crc_eof));
+	mfs = netdev->mtu - (sizeof(struct fcoe_hdr) +
+			     sizeof(struct fcoe_crc_eof));
 	if (fc_set_mfs(lp, mfs))
 		return -EINVAL;
 
 	/* offload features support */
-	if (fc->real_dev->features & NETIF_F_SG)
+	if (netdev->features & NETIF_F_SG)
 		lp->sg_supp = 1;
 
-#ifdef NETIF_F_FCOE_CRC
 	if (netdev->features & NETIF_F_FCOE_CRC) {
 		lp->crc_offload = 1;
 		FCOE_NETDEV_DBG(netdev, "Supports FCCRC offload\n");
 	}
-#endif
-#ifdef NETIF_F_FSO
 	if (netdev->features & NETIF_F_FSO) {
 		lp->seq_offload = 1;
 		lp->lso_max = netdev->gso_max_size;
 		FCOE_NETDEV_DBG(netdev, "Supports LSO for max len 0x%x\n",
 				lp->lso_max);
 	}
-#endif
 	if (netdev->fcoe_ddp_xid) {
 		lp->lro_enabled = 1;
 		lp->lro_xid = netdev->fcoe_ddp_xid;
 		FCOE_NETDEV_DBG(netdev, "Supports LRO for max xid 0x%x\n",
 				lp->lro_xid);
 	}
-	skb_queue_head_init(&fc->fcoe_pending_queue);
-	fc->fcoe_pending_queue_active = 0;
-	setup_timer(&fc->timer, fcoe_queue_timer, (unsigned long)lp);
-
-	/* look for SAN MAC address, if multiple SAN MACs exist, only
-	 * use the first one for SPMA */
-	rcu_read_lock();
-	for_each_dev_addr(netdev, ha) {
-		if ((ha->type == NETDEV_HW_ADDR_T_SAN) &&
-		    (is_valid_ether_addr(fc->ctlr.ctl_src_addr))) {
-			memcpy(fc->ctlr.ctl_src_addr, ha->addr, ETH_ALEN);
-			fc->ctlr.spma = 1;
-			break;
-		}
-	}
-	rcu_read_unlock();
-
-	/* setup Source Mac Address */
-	if (!fc->ctlr.spma)
-		memcpy(fc->ctlr.ctl_src_addr, fc->real_dev->dev_addr,
-		       fc->real_dev->addr_len);
+	skb_queue_head_init(&port->fcoe_pending_queue);
+	port->fcoe_pending_queue_active = 0;
+	setup_timer(&port->timer, fcoe_queue_timer, (unsigned long)lp);
 
-	wwnn = fcoe_wwn_from_mac(fc->real_dev->dev_addr, 1, 0);
+	wwnn = fcoe_wwn_from_mac(netdev->dev_addr, 1, 0);
 	fc_set_wwnn(lp, wwnn);
 	/* XXX - 3rd arg needs to be vlan id */
-	wwpn = fcoe_wwn_from_mac(fc->real_dev->dev_addr, 2, 0);
+	wwpn = fcoe_wwn_from_mac(netdev->dev_addr, 2, 0);
 	fc_set_wwpn(lp, wwpn);
 
-	/*
-	 * Add FCoE MAC address as second unicast MAC address
-	 * or enter promiscuous mode if not capable of listening
-	 * for multiple unicast MACs.
-	 */
-	rtnl_lock();
-	memcpy(flogi_maddr, (u8[6]) FC_FCOE_FLOGI_MAC, ETH_ALEN);
-	dev_unicast_add(fc->real_dev, flogi_maddr);
-	if (fc->ctlr.spma)
-		dev_unicast_add(fc->real_dev, fc->ctlr.ctl_src_addr);
-	dev_mc_add(fc->real_dev, FIP_ALL_ENODE_MACS, ETH_ALEN, 0);
-	rtnl_unlock();
-
-	/*
-	 * setup the receive function from ethernet driver
-	 * on the ethertype for the given device
-	 */
-	fc->fcoe_packet_type.func = fcoe_rcv;
-	fc->fcoe_packet_type.type = __constant_htons(ETH_P_FCOE);
-	fc->fcoe_packet_type.dev = fc->real_dev;
-	dev_add_pack(&fc->fcoe_packet_type);
-
-	fc->fip_packet_type.func = fcoe_fip_recv;
-	fc->fip_packet_type.type = htons(ETH_P_FIP);
-	fc->fip_packet_type.dev = fc->real_dev;
-	dev_add_pack(&fc->fip_packet_type);
-
 	return 0;
 }
 
@@ -415,86 +514,140 @@ static int fcoe_shost_config(struct fc_lport *lp, struct Scsi_Host *shost,
 	return 0;
 }
 
+/*
+ * fcoe_oem_match() - match for read types IO
+ * @fp: the fc_frame for new IO.
+ *
+ * Returns : true for read types IO, otherwise returns false.
+ */
+bool fcoe_oem_match(struct fc_frame *fp)
+{
+	return fc_fcp_is_read(fr_fsp(fp)) &&
+		(fr_fsp(fp)->data_len > fcoe_ddp_min);
+}
+
 /**
  * fcoe_em_config() - allocates em for this lport
- * @lp: the port that em is to allocated for
+ * @lp: the fcoe that em is to allocated for
  *
  * Returns : 0 on success
  */
 static inline int fcoe_em_config(struct fc_lport *lp)
 {
-	BUG_ON(lp->emp);
+	struct fcoe_port *port = lport_priv(lp);
+	struct fcoe_interface *fcoe = port->fcoe;
+	struct fcoe_interface *oldfcoe = NULL;
+	struct net_device *old_real_dev, *cur_real_dev;
+	u16 min_xid = FCOE_MIN_XID;
+	u16 max_xid = FCOE_MAX_XID;
 
-	lp->emp = fc_exch_mgr_alloc(lp, FC_CLASS_3,
-				    FCOE_MIN_XID, FCOE_MAX_XID);
-	if (!lp->emp)
+	/*
+	 * Check if need to allocate an em instance for
+	 * offload exchange ids to be shared across all VN_PORTs/lport.
+	 */
+	if (!lp->lro_enabled || !lp->lro_xid || (lp->lro_xid >= max_xid)) {
+		lp->lro_xid = 0;
+		goto skip_oem;
+	}
+
+	/*
+	 * Reuse existing offload em instance in case
+	 * it is already allocated on real eth device
+	 */
+	if (fcoe->netdev->priv_flags & IFF_802_1Q_VLAN)
+		cur_real_dev = vlan_dev_real_dev(fcoe->netdev);
+	else
+		cur_real_dev = fcoe->netdev;
+
+	list_for_each_entry(oldfcoe, &fcoe_hostlist, list) {
+		if (oldfcoe->netdev->priv_flags & IFF_802_1Q_VLAN)
+			old_real_dev = vlan_dev_real_dev(oldfcoe->netdev);
+		else
+			old_real_dev = oldfcoe->netdev;
+
+		if (cur_real_dev == old_real_dev) {
+			fcoe->oem = oldfcoe->oem;
+			break;
+		}
+	}
+
+	if (fcoe->oem) {
+		if (!fc_exch_mgr_add(lp, fcoe->oem, fcoe_oem_match)) {
+			printk(KERN_ERR "fcoe_em_config: failed to add "
+			       "offload em:%p on interface:%s\n",
+			       fcoe->oem, fcoe->netdev->name);
+			return -ENOMEM;
+		}
+	} else {
+		fcoe->oem = fc_exch_mgr_alloc(lp, FC_CLASS_3,
+					    FCOE_MIN_XID, lp->lro_xid,
+					    fcoe_oem_match);
+		if (!fcoe->oem) {
+			printk(KERN_ERR "fcoe_em_config: failed to allocate "
+			       "em for offload exches on interface:%s\n",
+			       fcoe->netdev->name);
+			return -ENOMEM;
+		}
+	}
+
+	/*
+	 * Exclude offload EM xid range from next EM xid range.
+	 */
+	min_xid += lp->lro_xid + 1;
+
+skip_oem:
+	if (!fc_exch_mgr_alloc(lp, FC_CLASS_3, min_xid, max_xid, NULL)) {
+		printk(KERN_ERR "fcoe_em_config: failed to "
+		       "allocate em on interface %s\n", fcoe->netdev->name);
 		return -ENOMEM;
+	}
 
 	return 0;
 }
 
 /**
  * fcoe_if_destroy() - FCoE software HBA tear-down function
- * @netdev: ptr to the associated net_device
- *
- * Returns: 0 if link is OK for use by FCoE.
+ * @lport: fc_lport to destroy
  */
-static int fcoe_if_destroy(struct net_device *netdev)
+static void fcoe_if_destroy(struct fc_lport *lport)
 {
-	struct fc_lport *lp = NULL;
-	struct fcoe_softc *fc;
-
-	BUG_ON(!netdev);
+	struct fcoe_port *port = lport_priv(lport);
+	struct fcoe_interface *fcoe = port->fcoe;
+	struct net_device *netdev = fcoe->netdev;
 
 	FCOE_NETDEV_DBG(netdev, "Destroying interface\n");
 
-	lp = fcoe_hostlist_lookup(netdev);
-	if (!lp)
-		return -ENODEV;
-
-	fc = lport_priv(lp);
-
 	/* Logout of the fabric */
-	fc_fabric_logoff(lp);
+	fc_fabric_logoff(lport);
 
-	/* Remove the instance from fcoe's list */
-	fcoe_hostlist_remove(lp);
+	/* Cleanup the fc_lport */
+	fc_lport_destroy(lport);
+	fc_fcp_destroy(lport);
 
-	/* clean up netdev configurations */
-	fcoe_netdev_cleanup(fc);
+	/* Stop the transmit retry timer */
+	del_timer_sync(&port->timer);
 
-	/* tear-down the FCoE controller */
-	fcoe_ctlr_destroy(&fc->ctlr);
+	/* Free existing transmit skbs */
+	fcoe_clean_pending_queue(lport);
 
-	/* Cleanup the fc_lport */
-	fc_lport_destroy(lp);
-	fc_fcp_destroy(lp);
+	/* receives may not be stopped until after this */
+	fcoe_interface_put(fcoe);
+
+	/* Free queued packets for the per-CPU receive threads */
+	fcoe_percpu_clean(lport);
 
 	/* Detach from the scsi-ml */
-	fc_remove_host(lp->host);
-	scsi_remove_host(lp->host);
+	fc_remove_host(lport->host);
+	scsi_remove_host(lport->host);
 
 	/* There are no more rports or I/O, free the EM */
-	if (lp->emp)
-		fc_exch_mgr_free(lp->emp);
-
-	/* Free the per-CPU receive threads */
-	fcoe_percpu_clean(lp);
-
-	/* Free existing skbs */
-	fcoe_clean_pending_queue(lp);
-
-	/* Stop the timer */
-	del_timer_sync(&fc->timer);
+	fc_exch_mgr_free(lport);
 
 	/* Free memory used by statistical counters */
-	fc_lport_free_stats(lp);
-
-	/* Release the net_device and Scsi_Host */
-	dev_put(fc->real_dev);
-	scsi_host_put(lp->host);
+	fc_lport_free_stats(lport);
 
-	return 0;
+	/* Release the Scsi_Host */
+	scsi_host_put(lport->host);
 }
 
 /*
@@ -540,106 +693,96 @@ static struct libfc_function_template fcoe_libfc_fcn_templ = {
 };
 
 /**
- * fcoe_if_create() - this function creates the fcoe interface
- * @netdev: pointer the associated netdevice
+ * fcoe_if_create() - this function creates the fcoe port
+ * @fcoe: fcoe_interface structure to create an fc_lport instance on
+ * @parent: device pointer to be the parent in sysfs for the SCSI host
  *
- * Creates fc_lport struct and scsi_host for lport, configures lport
- * and starts fabric login.
+ * Creates fc_lport struct and scsi_host for lport, configures lport.
  *
- * Returns : 0 on success
+ * Returns : The allocated fc_lport or an error pointer
  */
-static int fcoe_if_create(struct net_device *netdev)
+static struct fc_lport *fcoe_if_create(struct fcoe_interface *fcoe,
+				       struct device *parent)
 {
 	int rc;
-	struct fc_lport *lp = NULL;
-	struct fcoe_softc *fc;
+	struct fc_lport *lport = NULL;
+	struct fcoe_port *port;
 	struct Scsi_Host *shost;
-
-	BUG_ON(!netdev);
+	struct net_device *netdev = fcoe->netdev;
 
 	FCOE_NETDEV_DBG(netdev, "Create Interface\n");
 
-	lp = fcoe_hostlist_lookup(netdev);
-	if (lp)
-		return -EEXIST;
-
 	shost = libfc_host_alloc(&fcoe_shost_template,
-				 sizeof(struct fcoe_softc));
+				 sizeof(struct fcoe_port));
 	if (!shost) {
 		FCOE_NETDEV_DBG(netdev, "Could not allocate host structure\n");
-		return -ENOMEM;
+		rc = -ENOMEM;
+		goto out;
 	}
-	lp = shost_priv(shost);
-	fc = lport_priv(lp);
+	lport = shost_priv(shost);
+	port = lport_priv(lport);
+	port->lport = lport;
+	port->fcoe = fcoe;
+	INIT_WORK(&port->destroy_work, fcoe_destroy_work);
 
 	/* configure fc_lport, e.g., em */
-	rc = fcoe_lport_config(lp);
+	rc = fcoe_lport_config(lport);
 	if (rc) {
 		FCOE_NETDEV_DBG(netdev, "Could not configure lport for the "
 				"interface\n");
 		goto out_host_put;
 	}
 
-	/*
-	 * Initialize FIP.
-	 */
-	fcoe_ctlr_init(&fc->ctlr);
-	fc->ctlr.send = fcoe_fip_send;
-	fc->ctlr.update_mac = fcoe_update_src_mac;
-
 	/* configure lport network properties */
-	rc = fcoe_netdev_config(lp, netdev);
+	rc = fcoe_netdev_config(lport, netdev);
 	if (rc) {
 		FCOE_NETDEV_DBG(netdev, "Could not configure netdev for the "
 				"interface\n");
-		goto out_netdev_cleanup;
+		goto out_lp_destroy;
 	}
 
 	/* configure lport scsi host properties */
-	rc = fcoe_shost_config(lp, shost, &netdev->dev);
+	rc = fcoe_shost_config(lport, shost, parent);
 	if (rc) {
 		FCOE_NETDEV_DBG(netdev, "Could not configure shost for the "
 				"interface\n");
-		goto out_netdev_cleanup;
-	}
-
-	/* lport exch manager allocation */
-	rc = fcoe_em_config(lp);
-	if (rc) {
-		FCOE_NETDEV_DBG(netdev, "Could not configure the EM for the "
-				"interface\n");
-		goto out_netdev_cleanup;
+		goto out_lp_destroy;
 	}
 
 	/* Initialize the library */
-	rc = fcoe_libfc_config(lp, &fcoe_libfc_fcn_templ);
+	rc = fcoe_libfc_config(lport, &fcoe_libfc_fcn_templ);
 	if (rc) {
 		FCOE_NETDEV_DBG(netdev, "Could not configure libfc for the "
 				"interface\n");
 		goto out_lp_destroy;
 	}
 
-	/* add to lports list */
-	fcoe_hostlist_add(lp);
-
-	lp->boot_time = jiffies;
-
-	fc_fabric_login(lp);
-
-	if (!fcoe_link_ok(lp))
-		fcoe_ctlr_link_up(&fc->ctlr);
+	/*
+	 * fcoe_em_alloc() and fcoe_hostlist_add() both
+	 * need to be atomic with respect to other changes to the hostlist
+	 * since fcoe_em_alloc() looks for an existing EM
+	 * instance on host list updated by fcoe_hostlist_add().
+	 *
+	 * This is currently handled through the fcoe_config_mutex begin held.
+	 */
 
-	dev_hold(netdev);
+	/* lport exch manager allocation */
+	rc = fcoe_em_config(lport);
+	if (rc) {
+		FCOE_NETDEV_DBG(netdev, "Could not configure the EM for the "
+				"interface\n");
+		goto out_lp_destroy;
+	}
 
-	return rc;
+	fcoe_interface_get(fcoe);
+	return lport;
 
 out_lp_destroy:
-	fc_exch_mgr_free(lp->emp); /* Free the EM */
-out_netdev_cleanup:
-	fcoe_netdev_cleanup(fc);
+	fc_exch_mgr_free(lport);
 out_host_put:
-	scsi_host_put(lp->host);
-	return rc;
+	scsi_host_put(lport->host);
+out:
+	return ERR_PTR(rc);
 }
 
 /**
@@ -669,6 +812,7 @@ static int __init fcoe_if_init(void)
 int __exit fcoe_if_exit(void)
 {
 	fc_release_transport(scsi_transport_fcoe_sw);
+	scsi_transport_fcoe_sw = NULL;
 	return 0;
 }
 
@@ -686,7 +830,7 @@ static void fcoe_percpu_thread_create(unsigned int cpu)
 	thread = kthread_create(fcoe_percpu_receive_thread,
 				(void *)p, "fcoethread/%d", cpu);
 
-	if (likely(!IS_ERR(p->thread))) {
+	if (likely(!IS_ERR(thread))) {
 		kthread_bind(thread, cpu);
 		wake_up_process(thread);
 
@@ -838,14 +982,13 @@ int fcoe_rcv(struct sk_buff *skb, struct net_device *dev,
 {
 	struct fc_lport *lp;
 	struct fcoe_rcv_info *fr;
-	struct fcoe_softc *fc;
+	struct fcoe_interface *fcoe;
 	struct fc_frame_header *fh;
 	struct fcoe_percpu_s *fps;
-	unsigned short oxid;
-	unsigned int cpu = 0;
+	unsigned int cpu;
 
-	fc = container_of(ptype, struct fcoe_softc, fcoe_packet_type);
-	lp = fc->ctlr.lp;
+	fcoe = container_of(ptype, struct fcoe_interface, fcoe_packet_type);
+	lp = fcoe->ctlr.lp;
 	if (unlikely(lp == NULL)) {
 		FCOE_NETDEV_DBG(dev, "Cannot find hba structure");
 		goto err2;
@@ -876,20 +1019,20 @@ int fcoe_rcv(struct sk_buff *skb, struct net_device *dev,
 	skb_set_transport_header(skb, sizeof(struct fcoe_hdr));
 	fh = (struct fc_frame_header *) skb_transport_header(skb);
 
-	oxid = ntohs(fh->fh_ox_id);
-
 	fr = fcoe_dev_from_skb(skb);
 	fr->fr_dev = lp;
 	fr->ptype = ptype;
 
-#ifdef CONFIG_SMP
 	/*
-	 * The incoming frame exchange id(oxid) is ANDed with num of online
-	 * cpu bits to get cpu and then this cpu is used for selecting
-	 * a per cpu kernel thread from fcoe_percpu.
+	 * In case the incoming frame's exchange is originated from
+	 * the initiator, then received frame's exchange id is ANDed
+	 * with fc_cpu_mask bits to get the same cpu on which exchange
+	 * was originated, otherwise just use the current cpu.
 	 */
-	cpu = oxid & (num_online_cpus() - 1);
-#endif
+	if (ntoh24(fh->fh_f_ctl) & FC_FC_EX_CTX)
+		cpu = ntohs(fh->fh_ox_id) & fc_cpu_mask;
+	else
+		cpu = smp_processor_id();
 
 	fps = &per_cpu(fcoe_percpu, cpu);
 	spin_lock_bh(&fps->fcoe_rx_list.lock);
@@ -996,7 +1139,7 @@ static int fcoe_get_paged_crc_eof(struct sk_buff *skb, int tlen)
  * fcoe_fc_crc() - calculates FC CRC in this fcoe skb
  * @fp: the fc_frame containing data to be checksummed
  *
- * This uses crc32() to calculate the crc for fc frame
+ * This uses crc32() to calculate the crc for port frame
  * Return   : 32 bit crc
  */
 u32 fcoe_fc_crc(struct fc_frame *fp)
@@ -1029,7 +1172,7 @@ u32 fcoe_fc_crc(struct fc_frame *fp)
 
 /**
  * fcoe_xmit() - FCoE frame transmit function
- * @lp:	the associated local port
+ * @lp:	the associated local fcoe
  * @fp: the fc_frame to be transmitted
  *
  * Return   : 0 for success
@@ -1046,13 +1189,13 @@ int fcoe_xmit(struct fc_lport *lp, struct fc_frame *fp)
 	unsigned int hlen;		/* header length implies the version */
 	unsigned int tlen;		/* trailer length */
 	unsigned int elen;		/* eth header, may include vlan */
-	struct fcoe_softc *fc;
+	struct fcoe_port *port = lport_priv(lp);
+	struct fcoe_interface *fcoe = port->fcoe;
 	u8 sof, eof;
 	struct fcoe_hdr *hp;
 
 	WARN_ON((fr_len(fp) % sizeof(u32)) != 0);
 
-	fc = lport_priv(lp);
 	fh = fc_frame_header_get(fp);
 	skb = fp_skb(fp);
 	wlen = skb->len / FCOE_WORD_TO_BYTE;
@@ -1063,7 +1206,7 @@ int fcoe_xmit(struct fc_lport *lp, struct fc_frame *fp)
 	}
 
 	if (unlikely(fh->fh_r_ctl == FC_RCTL_ELS_REQ) &&
-	    fcoe_ctlr_els_send(&fc->ctlr, skb))
+	    fcoe_ctlr_els_send(&fcoe->ctlr, skb))
 		return 0;
 
 	sof = fr_sof(fp);
@@ -1085,7 +1228,7 @@ int fcoe_xmit(struct fc_lport *lp, struct fc_frame *fp)
 		crc = fcoe_fc_crc(fp);
 	}
 
-	/* copy fc crc and eof to the skb buff */
+	/* copy port crc and eof to the skb buff */
 	if (skb_is_nonlinear(skb)) {
 		skb_frag_t *frag;
 		if (fcoe_get_paged_crc_eof(skb, tlen)) {
@@ -1108,27 +1251,27 @@ int fcoe_xmit(struct fc_lport *lp, struct fc_frame *fp)
 		cp = NULL;
 	}
 
-	/* adjust skb network/transport offsets to match mac/fcoe/fc */
+	/* adjust skb network/transport offsets to match mac/fcoe/port */
 	skb_push(skb, elen + hlen);
 	skb_reset_mac_header(skb);
 	skb_reset_network_header(skb);
 	skb->mac_len = elen;
 	skb->protocol = htons(ETH_P_FCOE);
-	skb->dev = fc->real_dev;
+	skb->dev = fcoe->netdev;
 
 	/* fill up mac and fcoe headers */
 	eh = eth_hdr(skb);
 	eh->h_proto = htons(ETH_P_FCOE);
-	if (fc->ctlr.map_dest)
+	if (fcoe->ctlr.map_dest)
 		fc_fcoe_set_mac(eh->h_dest, fh->fh_d_id);
 	else
 		/* insert GW address */
-		memcpy(eh->h_dest, fc->ctlr.dest_addr, ETH_ALEN);
+		memcpy(eh->h_dest, fcoe->ctlr.dest_addr, ETH_ALEN);
 
-	if (unlikely(fc->ctlr.flogi_oxid != FC_XID_UNKNOWN))
-		memcpy(eh->h_source, fc->ctlr.ctl_src_addr, ETH_ALEN);
+	if (unlikely(fcoe->ctlr.flogi_oxid != FC_XID_UNKNOWN))
+		memcpy(eh->h_source, fcoe->ctlr.ctl_src_addr, ETH_ALEN);
 	else
-		memcpy(eh->h_source, fc->ctlr.data_src_addr, ETH_ALEN);
+		memcpy(eh->h_source, fcoe->ctlr.data_src_addr, ETH_ALEN);
 
 	hp = (struct fcoe_hdr *)(eh + 1);
 	memset(hp, 0, sizeof(*hp));
@@ -1136,7 +1279,6 @@ int fcoe_xmit(struct fc_lport *lp, struct fc_frame *fp)
 		FC_FCOE_ENCAPS_VER(hp, FC_FCOE_VER);
 	hp->fcoe_sof = sof;
 
-#ifdef NETIF_F_FSO
 	/* fcoe lso, mss is in max_payload which is non-zero for FCP data */
 	if (lp->seq_offload && fr_max_payload(fp)) {
 		skb_shinfo(skb)->gso_type = SKB_GSO_FCOE;
@@ -1145,7 +1287,6 @@ int fcoe_xmit(struct fc_lport *lp, struct fc_frame *fp)
 		skb_shinfo(skb)->gso_type = 0;
 		skb_shinfo(skb)->gso_size = 0;
 	}
-#endif
 	/* update tx stats: regardless if LLD fails */
 	stats = fc_lport_get_stats(lp);
 	stats->TxFrames++;
@@ -1153,7 +1294,7 @@ int fcoe_xmit(struct fc_lport *lp, struct fc_frame *fp)
 
 	/* send down to lld */
 	fr_dev(fp) = lp;
-	if (fc->fcoe_pending_queue.qlen)
+	if (port->fcoe_pending_queue.qlen)
 		fcoe_check_wait_queue(lp, skb);
 	else if (fcoe_start_io(skb))
 		fcoe_check_wait_queue(lp, skb);
@@ -1162,6 +1303,15 @@ int fcoe_xmit(struct fc_lport *lp, struct fc_frame *fp)
 }
 
 /**
+ * fcoe_percpu_flush_done() - Indicate percpu queue flush completion.
+ * @skb: the skb being completed.
+ */
+static void fcoe_percpu_flush_done(struct sk_buff *skb)
+{
+	complete(&fcoe_flush_completion);
+}
+
+/**
  * fcoe_percpu_receive_thread() - recv thread per cpu
  * @arg: ptr to the fcoe per cpu struct
  *
@@ -1179,7 +1329,7 @@ int fcoe_percpu_receive_thread(void *arg)
 	struct fcoe_crc_eof crc_eof;
 	struct fc_frame *fp;
 	u8 *mac = NULL;
-	struct fcoe_softc *fc;
+	struct fcoe_port *port;
 	struct fcoe_hdr *hp;
 
 	set_user_nice(current, -20);
@@ -1200,7 +1350,8 @@ int fcoe_percpu_receive_thread(void *arg)
 		fr = fcoe_dev_from_skb(skb);
 		lp = fr->fr_dev;
 		if (unlikely(lp == NULL)) {
-			FCOE_NETDEV_DBG(skb->dev, "Invalid HBA Structure");
+			if (skb->destructor != fcoe_percpu_flush_done)
+				FCOE_NETDEV_DBG(skb->dev, "NULL lport in skb");
 			kfree_skb(skb);
 			continue;
 		}
@@ -1215,7 +1366,7 @@ int fcoe_percpu_receive_thread(void *arg)
 		/*
 		 * Save source MAC address before discarding header.
 		 */
-		fc = lport_priv(lp);
+		port = lport_priv(lp);
 		if (skb_is_nonlinear(skb))
 			skb_linearize(skb);	/* not ideal */
 		mac = eth_hdr(skb)->h_source;
@@ -1277,7 +1428,7 @@ int fcoe_percpu_receive_thread(void *arg)
 		fh = fc_frame_header_get(fp);
 		if (fh->fh_r_ctl == FC_RCTL_DD_SOL_DATA &&
 		    fh->fh_type == FC_TYPE_FCP) {
-			fc_exch_recv(lp, lp->emp, fp);
+			fc_exch_recv(lp, fp);
 			continue;
 		}
 		if (fr_flags(fp) & FCPHF_CRC_UNCHECKED) {
@@ -1293,12 +1444,12 @@ int fcoe_percpu_receive_thread(void *arg)
 			}
 			fr_flags(fp) &= ~FCPHF_CRC_UNCHECKED;
 		}
-		if (unlikely(fc->ctlr.flogi_oxid != FC_XID_UNKNOWN) &&
-		    fcoe_ctlr_recv_flogi(&fc->ctlr, fp, mac)) {
+		if (unlikely(port->fcoe->ctlr.flogi_oxid != FC_XID_UNKNOWN) &&
+		    fcoe_ctlr_recv_flogi(&port->fcoe->ctlr, fp, mac)) {
 			fc_frame_free(fp);
 			continue;
 		}
-		fc_exch_recv(lp, lp->emp, fp);
+		fc_exch_recv(lp, fp);
 	}
 	return 0;
 }
@@ -1318,46 +1469,46 @@ int fcoe_percpu_receive_thread(void *arg)
  */
 static void fcoe_check_wait_queue(struct fc_lport *lp, struct sk_buff *skb)
 {
-	struct fcoe_softc *fc = lport_priv(lp);
+	struct fcoe_port *port = lport_priv(lp);
 	int rc;
 
-	spin_lock_bh(&fc->fcoe_pending_queue.lock);
+	spin_lock_bh(&port->fcoe_pending_queue.lock);
 
 	if (skb)
-		__skb_queue_tail(&fc->fcoe_pending_queue, skb);
+		__skb_queue_tail(&port->fcoe_pending_queue, skb);
 
-	if (fc->fcoe_pending_queue_active)
+	if (port->fcoe_pending_queue_active)
 		goto out;
-	fc->fcoe_pending_queue_active = 1;
+	port->fcoe_pending_queue_active = 1;
 
-	while (fc->fcoe_pending_queue.qlen) {
+	while (port->fcoe_pending_queue.qlen) {
 		/* keep qlen > 0 until fcoe_start_io succeeds */
-		fc->fcoe_pending_queue.qlen++;
-		skb = __skb_dequeue(&fc->fcoe_pending_queue);
+		port->fcoe_pending_queue.qlen++;
+		skb = __skb_dequeue(&port->fcoe_pending_queue);
 
-		spin_unlock_bh(&fc->fcoe_pending_queue.lock);
+		spin_unlock_bh(&port->fcoe_pending_queue.lock);
 		rc = fcoe_start_io(skb);
-		spin_lock_bh(&fc->fcoe_pending_queue.lock);
+		spin_lock_bh(&port->fcoe_pending_queue.lock);
 
 		if (rc) {
-			__skb_queue_head(&fc->fcoe_pending_queue, skb);
+			__skb_queue_head(&port->fcoe_pending_queue, skb);
 			/* undo temporary increment above */
-			fc->fcoe_pending_queue.qlen--;
+			port->fcoe_pending_queue.qlen--;
 			break;
 		}
 		/* undo temporary increment above */
-		fc->fcoe_pending_queue.qlen--;
+		port->fcoe_pending_queue.qlen--;
 	}
 
-	if (fc->fcoe_pending_queue.qlen < FCOE_LOW_QUEUE_DEPTH)
+	if (port->fcoe_pending_queue.qlen < FCOE_LOW_QUEUE_DEPTH)
 		lp->qfull = 0;
-	if (fc->fcoe_pending_queue.qlen && !timer_pending(&fc->timer))
-		mod_timer(&fc->timer, jiffies + 2);
-	fc->fcoe_pending_queue_active = 0;
+	if (port->fcoe_pending_queue.qlen && !timer_pending(&port->timer))
+		mod_timer(&port->timer, jiffies + 2);
+	port->fcoe_pending_queue_active = 0;
 out:
-	if (fc->fcoe_pending_queue.qlen > FCOE_MAX_QUEUE_DEPTH)
+	if (port->fcoe_pending_queue.qlen > FCOE_MAX_QUEUE_DEPTH)
 		lp->qfull = 1;
-	spin_unlock_bh(&fc->fcoe_pending_queue.lock);
+	spin_unlock_bh(&port->fcoe_pending_queue.lock);
 	return;
 }
 
@@ -1391,21 +1542,20 @@ static int fcoe_device_notification(struct notifier_block *notifier,
 				    ulong event, void *ptr)
 {
 	struct fc_lport *lp = NULL;
-	struct net_device *real_dev = ptr;
-	struct fcoe_softc *fc;
+	struct net_device *netdev = ptr;
+	struct fcoe_interface *fcoe;
+	struct fcoe_port *port;
 	struct fcoe_dev_stats *stats;
 	u32 link_possible = 1;
 	u32 mfs;
 	int rc = NOTIFY_OK;
 
-	read_lock(&fcoe_hostlist_lock);
-	list_for_each_entry(fc, &fcoe_hostlist, list) {
-		if (fc->real_dev == real_dev) {
-			lp = fc->ctlr.lp;
+	list_for_each_entry(fcoe, &fcoe_hostlist, list) {
+		if (fcoe->netdev == netdev) {
+			lp = fcoe->ctlr.lp;
 			break;
 		}
 	}
-	read_unlock(&fcoe_hostlist_lock);
 	if (lp == NULL) {
 		rc = NOTIFY_DONE;
 		goto out;
@@ -1420,21 +1570,27 @@ static int fcoe_device_notification(struct notifier_block *notifier,
 	case NETDEV_CHANGE:
 		break;
 	case NETDEV_CHANGEMTU:
-		mfs = fc->real_dev->mtu -
-			(sizeof(struct fcoe_hdr) +
-			 sizeof(struct fcoe_crc_eof));
+		mfs = netdev->mtu - (sizeof(struct fcoe_hdr) +
+				     sizeof(struct fcoe_crc_eof));
 		if (mfs >= FC_MIN_MAX_FRAME)
 			fc_set_mfs(lp, mfs);
 		break;
 	case NETDEV_REGISTER:
 		break;
+	case NETDEV_UNREGISTER:
+		list_del(&fcoe->list);
+		port = lport_priv(fcoe->ctlr.lp);
+		fcoe_interface_cleanup(fcoe);
+		schedule_work(&port->destroy_work);
+		goto out;
+		break;
 	default:
-		FCOE_NETDEV_DBG(real_dev, "Unknown event %ld "
+		FCOE_NETDEV_DBG(netdev, "Unknown event %ld "
 				"from netdev netlink\n", event);
 	}
 	if (link_possible && !fcoe_link_ok(lp))
-		fcoe_ctlr_link_up(&fc->ctlr);
-	else if (fcoe_ctlr_link_down(&fc->ctlr)) {
+		fcoe_ctlr_link_up(&fcoe->ctlr);
+	else if (fcoe_ctlr_link_down(&fcoe->ctlr)) {
 		stats = fc_lport_get_stats(lp);
 		stats->LinkFailureCount++;
 		fcoe_clean_pending_queue(lp);
@@ -1465,75 +1621,6 @@ static struct net_device *fcoe_if_to_netdev(const char *buffer)
 }
 
 /**
- * fcoe_netdev_to_module_owner() - finds out the driver module of the netdev
- * @netdev: the target netdev
- *
- * Returns: ptr to the struct module, NULL for failure
- */
-static struct module *
-fcoe_netdev_to_module_owner(const struct net_device *netdev)
-{
-	struct device *dev;
-
-	if (!netdev)
-		return NULL;
-
-	dev = netdev->dev.parent;
-	if (!dev)
-		return NULL;
-
-	if (!dev->driver)
-		return NULL;
-
-	return dev->driver->owner;
-}
-
-/**
- * fcoe_ethdrv_get() - Hold the Ethernet driver
- * @netdev: the target netdev
- *
- * Holds the Ethernet driver module by try_module_get() for
- * the corresponding netdev.
- *
- * Returns: 0 for success
- */
-static int fcoe_ethdrv_get(const struct net_device *netdev)
-{
-	struct module *owner;
-
-	owner = fcoe_netdev_to_module_owner(netdev);
-	if (owner) {
-		FCOE_NETDEV_DBG(netdev, "Hold driver module %s\n",
-				module_name(owner));
-		return  try_module_get(owner);
-	}
-	return -ENODEV;
-}
-
-/**
- * fcoe_ethdrv_put() - Release the Ethernet driver
- * @netdev: the target netdev
- *
- * Releases the Ethernet driver module by module_put for
- * the corresponding netdev.
- *
- * Returns: 0 for success
- */
-static int fcoe_ethdrv_put(const struct net_device *netdev)
-{
-	struct module *owner;
-
-	owner = fcoe_netdev_to_module_owner(netdev);
-	if (owner) {
-		FCOE_NETDEV_DBG(netdev, "Release driver module %s\n",
-				module_name(owner));
-		module_put(owner);
-		return 0;
-	}
-	return -ENODEV;
-}
-
-/**
  * fcoe_destroy() - handles the destroy from sysfs
  * @buffer: expected to be an eth if name
  * @kp: associated kernel param
@@ -1542,34 +1629,57 @@ static int fcoe_ethdrv_put(const struct net_device *netdev)
  */
 static int fcoe_destroy(const char *buffer, struct kernel_param *kp)
 {
-	int rc;
+	struct fcoe_interface *fcoe;
 	struct net_device *netdev;
+	int rc;
+
+	mutex_lock(&fcoe_config_mutex);
+#ifdef CONFIG_FCOE_MODULE
+	/*
+	 * Make sure the module has been initialized, and is not about to be
+	 * removed.  Module paramter sysfs files are writable before the
+	 * module_init function is called and after module_exit.
+	 */
+	if (THIS_MODULE->state != MODULE_STATE_LIVE) {
+		rc = -ENODEV;
+		goto out_nodev;
+	}
+#endif
 
 	netdev = fcoe_if_to_netdev(buffer);
 	if (!netdev) {
 		rc = -ENODEV;
 		goto out_nodev;
 	}
-	/* look for existing lport */
-	if (!fcoe_hostlist_lookup(netdev)) {
+
+	rtnl_lock();
+	fcoe = fcoe_hostlist_lookup_port(netdev);
+	if (!fcoe) {
+		rtnl_unlock();
 		rc = -ENODEV;
 		goto out_putdev;
 	}
-	rc = fcoe_if_destroy(netdev);
-	if (rc) {
-		printk(KERN_ERR "fcoe: Failed to destroy interface (%s)\n",
-		       netdev->name);
-		rc = -EIO;
-		goto out_putdev;
-	}
-	fcoe_ethdrv_put(netdev);
-	rc = 0;
+	list_del(&fcoe->list);
+	fcoe_interface_cleanup(fcoe);
+	rtnl_unlock();
+	fcoe_if_destroy(fcoe->ctlr.lp);
 out_putdev:
 	dev_put(netdev);
 out_nodev:
+	mutex_unlock(&fcoe_config_mutex);
 	return rc;
 }
 
+static void fcoe_destroy_work(struct work_struct *work)
+{
+	struct fcoe_port *port;
+
+	port = container_of(work, struct fcoe_port, destroy_work);
+	mutex_lock(&fcoe_config_mutex);
+	fcoe_if_destroy(port->lport);
+	mutex_unlock(&fcoe_config_mutex);
+}
+
 /**
  * fcoe_create() - Handles the create call from sysfs
  * @buffer: expected to be an eth if name
@@ -1580,41 +1690,84 @@ out_nodev:
 static int fcoe_create(const char *buffer, struct kernel_param *kp)
 {
 	int rc;
+	struct fcoe_interface *fcoe;
+	struct fc_lport *lport;
 	struct net_device *netdev;
 
+	mutex_lock(&fcoe_config_mutex);
+#ifdef CONFIG_FCOE_MODULE
+	/*
+	 * Make sure the module has been initialized, and is not about to be
+	 * removed.  Module paramter sysfs files are writable before the
+	 * module_init function is called and after module_exit.
+	 */
+	if (THIS_MODULE->state != MODULE_STATE_LIVE) {
+		rc = -ENODEV;
+		goto out_nodev;
+	}
+#endif
+
+	rtnl_lock();
 	netdev = fcoe_if_to_netdev(buffer);
 	if (!netdev) {
 		rc = -ENODEV;
 		goto out_nodev;
 	}
+
 	/* look for existing lport */
 	if (fcoe_hostlist_lookup(netdev)) {
 		rc = -EEXIST;
 		goto out_putdev;
 	}
-	fcoe_ethdrv_get(netdev);
 
-	rc = fcoe_if_create(netdev);
-	if (rc) {
+	fcoe = fcoe_interface_create(netdev);
+	if (!fcoe) {
+		rc = -ENOMEM;
+		goto out_putdev;
+	}
+
+	lport = fcoe_if_create(fcoe, &netdev->dev);
+	if (IS_ERR(lport)) {
 		printk(KERN_ERR "fcoe: Failed to create interface (%s)\n",
 		       netdev->name);
-		fcoe_ethdrv_put(netdev);
 		rc = -EIO;
-		goto out_putdev;
+		fcoe_interface_cleanup(fcoe);
+		goto out_free;
 	}
+
+	/* Make this the "master" N_Port */
+	fcoe->ctlr.lp = lport;
+
+	/* add to lports list */
+	fcoe_hostlist_add(lport);
+
+	/* start FIP Discovery and FLOGI */
+	lport->boot_time = jiffies;
+	fc_fabric_login(lport);
+	if (!fcoe_link_ok(lport))
+		fcoe_ctlr_link_up(&fcoe->ctlr);
+
 	rc = 0;
+out_free:
+	/*
+	 * Release from init in fcoe_interface_create(), on success lport
+	 * should be holding a reference taken in fcoe_if_create().
+	 */
+	fcoe_interface_put(fcoe);
 out_putdev:
 	dev_put(netdev);
 out_nodev:
+	rtnl_unlock();
+	mutex_unlock(&fcoe_config_mutex);
 	return rc;
 }
 
 module_param_call(create, fcoe_create, NULL, NULL, S_IWUSR);
 __MODULE_PARM_TYPE(create, "string");
-MODULE_PARM_DESC(create, "Create fcoe port using net device passed in.");
+MODULE_PARM_DESC(create, "Create fcoe fcoe using net device passed in.");
 module_param_call(destroy, fcoe_destroy, NULL, NULL, S_IWUSR);
 __MODULE_PARM_TYPE(destroy, "string");
-MODULE_PARM_DESC(destroy, "Destroy fcoe port");
+MODULE_PARM_DESC(destroy, "Destroy fcoe fcoe");
 
 /**
  * fcoe_link_ok() - Check if link is ok for the fc_lport
@@ -1632,37 +1785,40 @@ MODULE_PARM_DESC(destroy, "Destroy fcoe port");
  */
 int fcoe_link_ok(struct fc_lport *lp)
 {
-	struct fcoe_softc *fc = lport_priv(lp);
-	struct net_device *dev = fc->real_dev;
+	struct fcoe_port *port = lport_priv(lp);
+	struct net_device *dev = port->fcoe->netdev;
 	struct ethtool_cmd ecmd = { ETHTOOL_GSET };
-	int rc = 0;
 
-	if ((dev->flags & IFF_UP) && netif_carrier_ok(dev)) {
-		dev = fc->phys_dev;
-		if (dev->ethtool_ops->get_settings) {
-			dev->ethtool_ops->get_settings(dev, &ecmd);
-			lp->link_supported_speeds &=
-				~(FC_PORTSPEED_1GBIT | FC_PORTSPEED_10GBIT);
-			if (ecmd.supported & (SUPPORTED_1000baseT_Half |
-					      SUPPORTED_1000baseT_Full))
-				lp->link_supported_speeds |= FC_PORTSPEED_1GBIT;
-			if (ecmd.supported & SUPPORTED_10000baseT_Full)
-				lp->link_supported_speeds |=
-					FC_PORTSPEED_10GBIT;
-			if (ecmd.speed == SPEED_1000)
-				lp->link_speed = FC_PORTSPEED_1GBIT;
-			if (ecmd.speed == SPEED_10000)
-				lp->link_speed = FC_PORTSPEED_10GBIT;
-		}
-	} else
-		rc = -1;
+	if ((dev->flags & IFF_UP) && netif_carrier_ok(dev) &&
+	    (!dev_ethtool_get_settings(dev, &ecmd))) {
+		lp->link_supported_speeds &=
+			~(FC_PORTSPEED_1GBIT | FC_PORTSPEED_10GBIT);
+		if (ecmd.supported & (SUPPORTED_1000baseT_Half |
+				      SUPPORTED_1000baseT_Full))
+			lp->link_supported_speeds |= FC_PORTSPEED_1GBIT;
+		if (ecmd.supported & SUPPORTED_10000baseT_Full)
+			lp->link_supported_speeds |=
+				FC_PORTSPEED_10GBIT;
+		if (ecmd.speed == SPEED_1000)
+			lp->link_speed = FC_PORTSPEED_1GBIT;
+		if (ecmd.speed == SPEED_10000)
+			lp->link_speed = FC_PORTSPEED_10GBIT;
 
-	return rc;
+		return 0;
+	}
+	return -1;
 }
 
 /**
  * fcoe_percpu_clean() - Clear the pending skbs for an lport
  * @lp: the fc_lport
+ *
+ * Must be called with fcoe_create_mutex held to single-thread completion.
+ *
+ * This flushes the pending skbs by adding a new skb to each queue and
+ * waiting until they are all freed.  This assures us that not only are
+ * there no packets that will be handled by the lport, but also that any
+ * threads already handling packet have returned.
  */
 void fcoe_percpu_clean(struct fc_lport *lp)
 {
@@ -1687,7 +1843,25 @@ void fcoe_percpu_clean(struct fc_lport *lp)
 				kfree_skb(skb);
 			}
 		}
+
+		if (!pp->thread || !cpu_online(cpu)) {
+			spin_unlock_bh(&pp->fcoe_rx_list.lock);
+			continue;
+		}
+
+		skb = dev_alloc_skb(0);
+		if (!skb) {
+			spin_unlock_bh(&pp->fcoe_rx_list.lock);
+			continue;
+		}
+		skb->destructor = fcoe_percpu_flush_done;
+
+		__skb_queue_tail(&pp->fcoe_rx_list, skb);
+		if (pp->fcoe_rx_list.qlen == 1)
+			wake_up_process(pp->thread);
 		spin_unlock_bh(&pp->fcoe_rx_list.lock);
+
+		wait_for_completion(&fcoe_flush_completion);
 	}
 }
 
@@ -1699,16 +1873,16 @@ void fcoe_percpu_clean(struct fc_lport *lp)
  */
 void fcoe_clean_pending_queue(struct fc_lport *lp)
 {
-	struct fcoe_softc  *fc = lport_priv(lp);
+	struct fcoe_port  *port = lport_priv(lp);
 	struct sk_buff *skb;
 
-	spin_lock_bh(&fc->fcoe_pending_queue.lock);
-	while ((skb = __skb_dequeue(&fc->fcoe_pending_queue)) != NULL) {
-		spin_unlock_bh(&fc->fcoe_pending_queue.lock);
+	spin_lock_bh(&port->fcoe_pending_queue.lock);
+	while ((skb = __skb_dequeue(&port->fcoe_pending_queue)) != NULL) {
+		spin_unlock_bh(&port->fcoe_pending_queue.lock);
 		kfree_skb(skb);
-		spin_lock_bh(&fc->fcoe_pending_queue.lock);
+		spin_lock_bh(&port->fcoe_pending_queue.lock);
 	}
-	spin_unlock_bh(&fc->fcoe_pending_queue.lock);
+	spin_unlock_bh(&port->fcoe_pending_queue.lock);
 }
 
 /**
@@ -1725,24 +1899,21 @@ int fcoe_reset(struct Scsi_Host *shost)
 }
 
 /**
- * fcoe_hostlist_lookup_softc() - find the corresponding lport by a given device
+ * fcoe_hostlist_lookup_port() - find the corresponding lport by a given device
  * @dev: this is currently ptr to net_device
  *
- * Returns: NULL or the located fcoe_softc
+ * Returns: NULL or the located fcoe_port
+ * Locking: must be called with the RNL mutex held
  */
-static struct fcoe_softc *
-fcoe_hostlist_lookup_softc(const struct net_device *dev)
+static struct fcoe_interface *
+fcoe_hostlist_lookup_port(const struct net_device *dev)
 {
-	struct fcoe_softc *fc;
+	struct fcoe_interface *fcoe;
 
-	read_lock(&fcoe_hostlist_lock);
-	list_for_each_entry(fc, &fcoe_hostlist, list) {
-		if (fc->real_dev == dev) {
-			read_unlock(&fcoe_hostlist_lock);
-			return fc;
-		}
+	list_for_each_entry(fcoe, &fcoe_hostlist, list) {
+		if (fcoe->netdev == dev)
+			return fcoe;
 	}
-	read_unlock(&fcoe_hostlist_lock);
 	return NULL;
 }
 
@@ -1751,14 +1922,14 @@ fcoe_hostlist_lookup_softc(const struct net_device *dev)
  * @netdev: ptr to net_device
  *
  * Returns: 0 for success
+ * Locking: must be called with the RTNL mutex held
  */
-struct fc_lport *fcoe_hostlist_lookup(const struct net_device *netdev)
+static struct fc_lport *fcoe_hostlist_lookup(const struct net_device *netdev)
 {
-	struct fcoe_softc *fc;
-
-	fc = fcoe_hostlist_lookup_softc(netdev);
+	struct fcoe_interface *fcoe;
 
-	return (fc) ? fc->ctlr.lp : NULL;
+	fcoe = fcoe_hostlist_lookup_port(netdev);
+	return (fcoe) ? fcoe->ctlr.lp : NULL;
 }
 
 /**
@@ -1766,41 +1937,23 @@ struct fc_lport *fcoe_hostlist_lookup(const struct net_device *netdev)
  * @lp: ptr to the fc_lport to be added
  *
  * Returns: 0 for success
+ * Locking: must be called with the RTNL mutex held
  */
-int fcoe_hostlist_add(const struct fc_lport *lp)
+static int fcoe_hostlist_add(const struct fc_lport *lport)
 {
-	struct fcoe_softc *fc;
-
-	fc = fcoe_hostlist_lookup_softc(fcoe_netdev(lp));
-	if (!fc) {
-		fc = lport_priv(lp);
-		write_lock_bh(&fcoe_hostlist_lock);
-		list_add_tail(&fc->list, &fcoe_hostlist);
-		write_unlock_bh(&fcoe_hostlist_lock);
+	struct fcoe_interface *fcoe;
+	struct fcoe_port *port;
+
+	fcoe = fcoe_hostlist_lookup_port(fcoe_netdev(lport));
+	if (!fcoe) {
+		port = lport_priv(lport);
+		fcoe = port->fcoe;
+		list_add_tail(&fcoe->list, &fcoe_hostlist);
 	}
 	return 0;
 }
 
 /**
- * fcoe_hostlist_remove() - remove a lport from lports list
- * @lp: ptr to the fc_lport to be removed
- *
- * Returns: 0 for success
- */
-int fcoe_hostlist_remove(const struct fc_lport *lp)
-{
-	struct fcoe_softc *fc;
-
-	fc = fcoe_hostlist_lookup_softc(fcoe_netdev(lp));
-	BUG_ON(!fc);
-	write_lock_bh(&fcoe_hostlist_lock);
-	list_del(&fc->list);
-	write_unlock_bh(&fcoe_hostlist_lock);
-
-	return 0;
-}
-
-/**
  * fcoe_init() - fcoe module loading initialization
  *
  * Returns 0 on success, negative on failure
@@ -1811,8 +1964,7 @@ static int __init fcoe_init(void)
 	int rc = 0;
 	struct fcoe_percpu_s *p;
 
-	INIT_LIST_HEAD(&fcoe_hostlist);
-	rwlock_init(&fcoe_hostlist_lock);
+	mutex_lock(&fcoe_config_mutex);
 
 	for_each_possible_cpu(cpu) {
 		p = &per_cpu(fcoe_percpu, cpu);
@@ -1830,15 +1982,18 @@ static int __init fcoe_init(void)
 	/* Setup link change notification */
 	fcoe_dev_setup();
 
-	fcoe_if_init();
+	rc = fcoe_if_init();
+	if (rc)
+		goto out_free;
 
+	mutex_unlock(&fcoe_config_mutex);
 	return 0;
 
 out_free:
 	for_each_online_cpu(cpu) {
 		fcoe_percpu_thread_destroy(cpu);
 	}
-
+	mutex_unlock(&fcoe_config_mutex);
 	return rc;
 }
 module_init(fcoe_init);
@@ -1851,21 +2006,36 @@ module_init(fcoe_init);
 static void __exit fcoe_exit(void)
 {
 	unsigned int cpu;
-	struct fcoe_softc *fc, *tmp;
+	struct fcoe_interface *fcoe, *tmp;
+	struct fcoe_port *port;
+
+	mutex_lock(&fcoe_config_mutex);
 
 	fcoe_dev_cleanup();
 
 	/* releases the associated fcoe hosts */
-	list_for_each_entry_safe(fc, tmp, &fcoe_hostlist, list)
-		fcoe_if_destroy(fc->real_dev);
+	rtnl_lock();
+	list_for_each_entry_safe(fcoe, tmp, &fcoe_hostlist, list) {
+		list_del(&fcoe->list);
+		port = lport_priv(fcoe->ctlr.lp);
+		fcoe_interface_cleanup(fcoe);
+		schedule_work(&port->destroy_work);
+	}
+	rtnl_unlock();
 
 	unregister_hotcpu_notifier(&fcoe_cpu_notifier);
 
-	for_each_online_cpu(cpu) {
+	for_each_online_cpu(cpu)
 		fcoe_percpu_thread_destroy(cpu);
-	}
 
-	/* detach from scsi transport */
+	mutex_unlock(&fcoe_config_mutex);
+
+	/* flush any asyncronous interface destroys,
+	 * this should happen after the netdev notifier is unregistered */
+	flush_scheduled_work();
+
+	/* detach from scsi transport
+	 * must happen after all destroys are done, therefor after the flush */
 	fcoe_if_exit();
 }
 module_exit(fcoe_exit);